@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 1,485 lines 59 KB
Raw
1 /**
2 * `alchemy dev` stress suite.
3 *
4 * Every other dev test in this repo asserts that a stack CONVERGES. This
5 * one asserts that the dev server SURVIVES — that no sequence of edits a
6 * developer can perform while `alchemy dev` is running kills it, wedges it,
7 * or leaves it serving stale code.
8 *
9 * Topology under test (one cross-cloud stack, see ../alchemy.run.ts):
10 *
11 * AWS (floci emulator) Cloudflare (workerd + docker)
12 * ──────────────────── ─────────────────────────────
13 * Lambda ApiFunction ◀──────── Worker ApiWorker (path `main`)
14 * ├ S3 StressBucket cross- Worker EchoWorker (Effect-native `main`)
15 * ├ DynamoDB Table cloud ├ KV + R2 + Durable Object
16 * └ SQS + consumer hop └ Container SandboxContainer
17 * Website.StaticSite Website.StaticSite (build mode)
18 * (dev-command child)
19 *
20 * The suite runs ONE dev server for its whole lifetime and accumulates
21 * churn on it — that accumulation IS the stress. Phases run in order and
22 * each one leaves the stack healthy for the next.
23 *
24 * Two distinct reload paths are told apart throughout by
25 * `server.planCount`, which counts `Plan:` renders (i.e. exec-child
26 * restarts). Both paths put new code in front of a request within ~2s —
27 * the local Worker provider's bundler watch loop always wins that race —
28 * so the discriminator is what happens AFTER the swap:
29 *
30 * - a file only the BUNDLER sees → hot swap, and `planCount` never
31 * (`src/api/marker.ts`) moves, however long you wait.
32 * - a file the STACK imports → hot swap, and then `bun --watch`
33 * (`src/echo/marker.ts`) re-runs the stack (~4s), replan,
34 * re-apply. `planCount` goes UP.
35 *
36 * A failed rebuild takes its own Worker off the air until the next good
37 * save; what must never happen is the CLI dying or an UNRELATED resource
38 * going with it. That is what the resilience phases assert.
39 *
40 * Requires Docker: the AWS half runs in floci and the Cloudflare Container
41 * runs as a real container. No cloud credentials are used or needed.
42 */
43 import { afterAll, beforeAll, expect, test } from "bun:test";
44 import * as fs from "node:fs";
45 import { PORTS } from "../src/ports.ts";
46 import {
47 at,
48 DevServer,
49 fetchJson,
50 fetchOk,
51 freePinnedPorts,
52 makeScratchProject,
53 PollTimeout,
54 resetFlociEmulator,
55 pollUntil,
56 waitForJson,
57 } from "./harness.ts";
58 import {
59 echoInboxClass,
60 echoQueueBindings,
61 echoQueueLayers,
62 echoQueueRoutes,
63 echoStreamImport,
64 extraWorkerDeclaration,
65 extraWorkerOutput,
66 extraWorkerSource,
67 lambdaArchiveBindings,
68 lambdaArchiveRoutes,
69 portSquatterDeclaration,
70 portSquatterSource,
71 reportFunctionDeclaration,
72 reportFunctionImport,
73 reportFunctionOutput,
74 reportFunctionSource,
75 secondImageBindings,
76 secondImageImport,
77 secondImageLayer,
78 secondImageRoutes,
79 secondImageSource,
80 secondImageWorkerImport,
81 } from "./mutations.ts";
82
83 const STAGE = "dev-stress";
84
85 // Long budgets: the first apply builds a Lambda bundle and a MicroVM
86 // image, boots floci, starts four workerd instances, runs a site build,
87 // and pulls a container image. Later phases are fast because only the
88 // delta re-applies.
89 //
90 // These are deliberately LARGER than the sum of every bounded poll inside
91 // a phase. `bun test` kills every child process the file spawned the
92 // moment a test hits its timeout (`killed N dangling processes`) — which
93 // would take the dev server down and turn one slow assertion into a
94 // cascade of meaningless failures in every later phase. A phase must
95 // therefore always fail through a `pollUntil` timeout (server still
96 // alive, output tail attached), never through bun's.
97 const BOOT_TIMEOUT = 1_800_000;
98 const PHASE_TIMEOUT = 1_800_000;
99
100 let server: DevServer;
101 let cfSiteUrl: string;
102 /** Cursor into the CLI output, advanced past phases that expect errors. */
103 let cleanCursor = 0;
104
105 /** URL on a floci host-routed address (`*.localhost.floci.io`). */
106 const at2 = (host: string, port: number, path: string) => new URL(path, `http://${host}:${port}`);
107
108 const echo = (path: string) => at(PORTS.echo, path);
109 const api = (path: string) => at(PORTS.api, path);
110 const extra = (path: string) => at(PORTS.extra, path);
111 const awsSite = (path: string) => at(PORTS.awsSite, path);
112 const extraAlt = (path: string) => at(PORTS.extraAlt, path);
113 const microvm = (path: string) => at(PORTS.microvm, path);
114 const ecs = (path: string) => at(PORTS.ecs, path);
115 const ecsInline = (path: string) => at(PORTS.ecsInline, path);
116
117 /**
118 * Poll an ECS-served URL until its TEXT body matches. The busybox httpd
119 * containers serve plain files; a container mid-roll refuses connections.
120 */
121 const waitForText = async (
122 what: string,
123 url: URL,
124 expected: string,
125 options?: { tries?: number; delayMs?: number },
126 ): Promise<void> => {
127 let last = "no response yet";
128 try {
129 await pollUntil(
130 what,
131 async () => {
132 try {
133 const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
134 const text = (await res.text()).trim();
135 last = `${res.status} ${text.slice(0, 200)}`;
136 return res.ok && text === expected ? true : undefined;
137 } catch (cause) {
138 last = `error: ${String(cause).slice(0, 200)}`;
139 return undefined;
140 }
141 },
142 { tries: 240, delayMs: 1_000, server, ...options },
143 );
144 } catch (cause) {
145 if (cause instanceof PollTimeout) {
146 cause.message = `${cause.message.split("\n")[0]}\n--- last response from ${url} ---\n${last}\n${cause.message.split("\n").slice(1).join("\n")}`;
147 }
148 throw cause;
149 }
150 };
151
152 /** Source text for a marker module — the suite's unit of observable change. */
153 const markerModule = (name: string, value: string) =>
154 `export const ${name} = ${JSON.stringify(value)};\n`;
155
156 let lambdaMarkerSeq = 0;
157 /**
158 * Bump the Lambda's marker and wait for the emulator to serve it, observed
159 * THROUGH the cross-cloud hop. Called after every phase that makes the
160 * engine re-reconcile the Function, because that is exactly where hot
161 * swap used to die: an engine update re-points the function's code at a
162 * content-addressed S3 key, detaching it from the watch loop's stable dev
163 * key, and the next marker edit was silently ignored until the next engine
164 * update (fixed in FlociFunctionProvider — this pins it).
165 */
166 const expectLambdaHotSwapStillWorks = async (context: string) => {
167 const marker = `lambda-after-${context}-${++lambdaMarkerSeq}`;
168 server.write("src/lambda/marker.ts", markerModule("LAMBDA_MARKER", marker));
169 await waitForJson<{ marker: string }>(
170 `the Lambda to hot-swap to ${marker} (${context})`,
171 api("/aws/"),
172 (body) => body.marker === marker,
173 { tries: 120, delayMs: 1_000, server },
174 );
175 };
176
177 const messageModule = (value: string) => `export const message = () => ${JSON.stringify(value)};\n`;
178
179 /** Is a port refusing connections (i.e. nothing is serving there)? */
180 const isClosed = async (url: URL) => {
181 try {
182 await fetch(url, { signal: AbortSignal.timeout(2_000) });
183 return false;
184 } catch {
185 return true;
186 }
187 };
188
189 /**
190 * Has `url` stopped serving the worker that used to answer with `marker`?
191 * A deleted local Worker normally closes its port, but a lingering dev
192 * proxy answering 5xx counts as "gone" too — what matters is that the
193 * removed code is no longer reachable.
194 */
195 const stoppedServing = async (url: URL, marker: string) => {
196 try {
197 const res = await fetch(url, { signal: AbortSignal.timeout(2_000) });
198 if (!res.ok) return true;
199 const body = (await res.json()) as { marker?: string };
200 return body.marker !== marker;
201 } catch {
202 return true;
203 }
204 };
205
206 beforeAll(async () => {
207 resetFlociEmulator();
208 freePinnedPorts(Object.values(PORTS));
209 const cwd = makeScratchProject(STAGE);
210 server = new DevServer({ cwd, stage: STAGE });
211
212 // The first apply is done when the CLI prints its `Done:` summary.
213 await pollUntil(
214 "the first dev apply to complete",
215 () => (server.doneCount >= 1 ? true : undefined),
216 { tries: 1_700, delayMs: 1_000, server },
217 );
218 }, BOOT_TIMEOUT);
219
220 afterAll(async () => {
221 if (!server) return;
222 await server.shutdown();
223 if (!process.env.NO_DESTROY) {
224 server.destroyStack();
225 if (!process.env.DEBUG) {
226 fs.rmSync(server.cwd, { recursive: true, force: true });
227 }
228 }
229 }, 300_000);
230
231 // ───────────────────────────────────────────────────────────────────────
232 // Phase 1 — the stack converges, locally, across both clouds.
233 // ───────────────────────────────────────────────────────────────────────
234
235 test(
236 "boots: every local resource serves, and the cross-cloud hop works",
237 async () => {
238 server.assertAlive("boot");
239
240 // ── Cloudflare, Effect-native worker ──
241 expect(await fetchJson<{ marker: string }>(echo("/marker"))).toEqual({
242 marker: "echo-v1",
243 });
244
245 expect(await fetchJson<{ value: string }>(echo("/kv?key=boot"))).toEqual({
246 value: "kv:boot",
247 });
248
249 const r2 = await fetchJson<{ text: string; keys: string[] }>(echo("/r2"));
250 expect(r2.text).toBe("hello from r2");
251 expect(r2.keys).toContain("hello.txt");
252
253 // Durable Object state is real state, not a stub.
254 const first = await fetchJson<{ count: number }>(echo("/counter"));
255 const second = await fetchJson<{ count: number }>(echo("/counter"));
256 expect(second.count).toBe(first.count + 1);
257
258 // ── Cloudflare Container (docker), reached through the DO ──
259 const sandbox = await fetchJson<{ greeting: string; marker: string }>(
260 echo("/sandbox"),
261 undefined,
262 // The image may still be pulling/starting on the first request.
263 { tries: 180, delayMs: 1_000 },
264 );
265 expect(sandbox.greeting).toBe("hello-from-container");
266 expect(sandbox.marker).toBe("sandbox-v1");
267
268 // ── #1334: the container reaches a service on the HOST through an env
269 // var written as `http://localhost:…` (the dev runtime rewrites the
270 // loopback host to `host.docker.localhost`; Linux unix-socket-tunnels
271 // it into the sidecar netns instead of SYNing the docker bridge) ──
272 const hostFetch = await fetchJson<{ target: string; body: string }>(
273 echo("/sandbox/host-fetch"),
274 undefined,
275 { tries: 60, delayMs: 1_000 },
276 );
277 expect(hostFetch.target).toBe("http://host.docker.localhost:8793");
278 expect(hostFetch.body).toContain("aws-site-env-v1");
279
280 // ── Cloudflare, path-`main` worker ──
281 expect(await fetchJson<{ marker: string; message: string }>(api("/marker"))).toEqual({
282 marker: "api-v1",
283 message: "message-v1",
284 });
285
286 const env = await fetchJson<{
287 API_VARIABLE: string;
288 AWS_LAMBDA_URL: string;
289 }>(api("/env"));
290 expect(env.API_VARIABLE).toBe("api-variable-v1");
291 // Local identity: the Lambda URL is the emulator's, not real AWS.
292 expect(env.AWS_LAMBDA_URL).toContain("localhost:4566");
293
294 // ── CROSS-CLOUD: local workerd → the floci-hosted Lambda URL ──
295 const lambda = await fetchJson<{ marker: string; variable: string }>(api("/aws/"), undefined, {
296 tries: 120,
297 delayMs: 1_000,
298 });
299 expect(lambda).toEqual({
300 marker: "lambda-v1",
301 variable: "lambda-variable-v1",
302 });
303
304 // ── AWS data plane, driven through the cross-cloud hop ──
305 expect(await fetchJson<{ text: string }>(api("/aws/s3"))).toEqual({
306 text: "hello from s3",
307 });
308 expect(await fetchJson<{ text: string }>(api("/aws/dynamo"))).toEqual({
309 text: "hello from dynamo",
310 });
311
312 // SQS produce → floci poller → the same Lambda's consumer → DynamoDB.
313 const message = { id: crypto.randomUUID() };
314 await fetchOk(api("/aws/queue/send"), {
315 method: "POST",
316 body: JSON.stringify(message),
317 });
318 const delivered = await waitForJson<{ body: string | null }>(
319 "the queue message to be consumed",
320 api(`/aws/queue/messages?id=${message.id}`),
321 (body) => body.body !== null,
322 { tries: 120, delayMs: 500, server },
323 );
324 expect(JSON.parse(delivered.body!)).toEqual(message);
325
326 // ── AWS ECS: floci runs the services' tasks as real containers on
327 // the host daemon; bridge networking publishes the baked ports. First
328 // contact can wait out the image build + scheduler launch.
329 await waitForText("the context-built ECS service to serve", ecs("/"), "ecs-site-v1", {
330 tries: 300,
331 });
332 await waitForText(
333 "the ECS service's baked Dockerfile marker",
334 ecs("/baked.txt"),
335 "dockerfile-v1",
336 );
337 await waitForText("the ECS service's task-definition env", ecs("/env.txt"), "ecs-env-v1");
338 await waitForText(
339 "the inline-Dockerfile ECS service to serve",
340 ecsInline("/"),
341 "ecs-inline-v1",
342 { tries: 300 },
343 );
344
345 // ── AWS Website: the dev-command child, on its pinned port ──
346 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
347 const siteEnv = await fetchJson<{ marker: string; pid: number }>(awsSite("/__dev-env"));
348 expect(siteEnv.marker).toBe("aws-site-env-v1");
349
350 // ── Cloudflare Website: build mode, served by a local Worker ──
351 cfSiteUrl = await pollUntil(
352 "cfSiteUrl in the stack outputs",
353 () => server.outputUrl("cfSiteUrl"),
354 { tries: 60, delayMs: 1_000, server },
355 );
356 expect(cfSiteUrl).toMatch(/^http:\/\/localhost:\d+/);
357 expect(await (await fetchOk(new URL("/", cfSiteUrl))).text()).toContain("cf-site-v1");
358
359 // Nothing in a clean boot may look like a failure.
360 expect(server.output).not.toContain("alchemy dev: run failed");
361 expect(server.output).not.toContain("alchemy dev: apply failed");
362 cleanCursor = server.mark();
363 },
364 PHASE_TIMEOUT,
365 );
366
367 test(
368 "cross-cloud: a Cloudflare Worker boots an AWS Lambda MicroVM and drives both its protocols",
369 async () => {
370 // Boot → wait for RUNNING → auth token → typed RPC + raw HTTPS →
371 // terminate, all from local workerd against the floci emulator.
372 const roundtrip = await fetchJson<{
373 microvmId: string;
374 reply: string;
375 marker: string;
376 echo: string;
377 }>(microvm("/roundtrip?message=stress"), undefined, {
378 tries: 240,
379 delayMs: 1_000,
380 });
381 expect(roundtrip.microvmId).toBeTruthy();
382 expect(roundtrip.reply).toBe("hello, stress!");
383 expect(roundtrip.marker).toBe("vm-v1");
384 expect(roundtrip.echo).toBe("stress");
385 server.assertAlive("microvm roundtrip");
386 },
387 PHASE_TIMEOUT,
388 );
389
390 // ───────────────────────────────────────────────────────────────────────
391 // Phase 2 — the two reload paths, told apart.
392 // ───────────────────────────────────────────────────────────────────────
393
394 test(
395 "hot reload (bundler path): editing a file the stack never imports swaps the script without re-running the stack",
396 async () => {
397 const plansBefore = server.planCount;
398 const counterBefore = (await fetchJson<{ count: number }>(echo("/counter"))).count;
399
400 server.write("src/api/marker.ts", markerModule("API_MARKER", "api-v2"));
401
402 await waitForJson<{ marker: string }>(
403 "ApiWorker to serve api-v2",
404 api("/marker"),
405 (body) => body.marker === "api-v2",
406 { server },
407 );
408
409 // The whole point of this path: the stack process never re-ran. A
410 // re-plan would trail the swap by a couple of seconds, so settle well
411 // past that before claiming it never came.
412 await Bun.sleep(20_000);
413 expect(server.planCount).toBe(plansBefore);
414 server.assertAlive("bundler hot swap");
415
416 // And nothing else moved: the sibling worker's DO kept its state.
417 const counterAfter = (await fetchJson<{ count: number }>(echo("/counter"))).count;
418 expect(counterAfter).toBe(counterBefore + 1);
419 },
420 PHASE_TIMEOUT,
421 );
422
423 test(
424 "hot reload (watch path): editing a file the stack imports re-runs the stack and keeps sidecar children alive",
425 async () => {
426 const plansBefore = server.planCount;
427 // The dev-command child lives in the provider sidecar, which is meant
428 // to survive user-code restarts. Its pid is the proof.
429 const sitePidBefore = (await fetchJson<{ pid: number }>(awsSite("/__dev-env"))).pid;
430
431 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v2"));
432
433 await waitForJson<{ marker: string }>(
434 "EchoWorker to serve echo-v2",
435 echo("/marker"),
436 (body) => body.marker === "echo-v2",
437 { server },
438 );
439
440 // The re-plan trails the hot swap — wait for it rather than sampling.
441 await server.waitForPlanAfter(plansBefore);
442 server.assertAlive("watch-path reload");
443
444 // The sidecar-hosted dev server was NOT bounced by a user-code reload.
445 const sitePidAfter = (await fetchJson<{ pid: number }>(awsSite("/__dev-env"))).pid;
446 expect(sitePidAfter).toBe(sitePidBefore);
447
448 // The bundler-path worker was not disturbed either.
449 expect((await fetchJson<{ marker: string }>(api("/marker"))).marker).toBe("api-v2");
450 },
451 PHASE_TIMEOUT,
452 );
453
454 test(
455 "hot reload (AWS): editing the Lambda's source hot-swaps its code in the emulator",
456 async () => {
457 server.write("src/lambda/marker.ts", markerModule("LAMBDA_MARKER", "lambda-v2"));
458
459 // Observed THROUGH the cross-cloud hop, so this also re-proves that the
460 // Worker → Lambda edge still resolves after both sides reloaded.
461 await waitForJson<{ marker: string }>(
462 "the Lambda to serve lambda-v2",
463 api("/aws/"),
464 (body) => body.marker === "lambda-v2",
465 { tries: 240, delayMs: 1_000, server },
466 );
467
468 server.assertAlive("lambda hot swap");
469 // Bindings survived the swap.
470 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe("hello from s3");
471 },
472 PHASE_TIMEOUT,
473 );
474
475 // ───────────────────────────────────────────────────────────────────────
476 // Phase 3 — files that move out from under the bundler.
477 // ───────────────────────────────────────────────────────────────────────
478
479 test(
480 "surviving a moved module: the importer points at a missing path, then the file arrives",
481 async () => {
482 const importer = server.read("src/api-worker.ts");
483
484 // Deliberately out of order: rewrite the import FIRST, so the bundler
485 // is asked to resolve a module that does not exist yet.
486 server.write(
487 "src/api-worker.ts",
488 importer.replace('"./api/message.ts"', '"./api/nested/message.ts"'),
489 );
490 await Bun.sleep(8_000);
491 // A failed rebuild takes ITS worker off the air — that part is fine.
492 // What must hold is that the dev server lives and every unrelated
493 // resource is untouched.
494 server.assertAlive("broken import window");
495 expect((await fetchJson<{ marker: string }>(echo("/marker"))).marker).toBe("echo-v2");
496 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
497
498 // Complete the move, with a NEW value so "the rebuild happened" cannot
499 // be confused with "the old bundle is still serving".
500 server.remove("src/api/message.ts");
501 server.write("src/api/nested/message.ts", messageModule("message-moved"));
502 await waitForJson<{ message: string }>(
503 "ApiWorker to rebuild against the moved module",
504 api("/marker"),
505 (body) => body.message === "message-moved",
506 { tries: 180, delayMs: 500, server },
507 );
508
509 // The NEW path is watched too, not just resolved once.
510 server.write("src/api/nested/message.ts", messageModule("message-v2"));
511 await waitForJson<{ message: string }>(
512 "the moved module's edit to land",
513 api("/marker"),
514 (body) => body.message === "message-v2",
515 { server },
516 );
517 server.assertAlive("moved module reload");
518 },
519 PHASE_TIMEOUT,
520 );
521
522 // ───────────────────────────────────────────────────────────────────────
523 // Phase 4 — broken states. The dev server must log and wait, never exit.
524 // ───────────────────────────────────────────────────────────────────────
525
526 test(
527 "surviving a syntax error in a stack-imported module",
528 async () => {
529 const plansBefore = server.planCount;
530 const cursor = server.mark();
531 // Unterminated string literal: the module cannot even be parsed, so
532 // importing the stack throws before any Alchemy code runs.
533 server.write("src/echo/marker.ts", 'export const ECHO_MARKER = "oops\n');
534
535 await pollUntil(
536 "the CLI to report the failed run",
537 () =>
538 /alchemy dev: run failed|SyntaxError|Unterminated|Unexpected end/.test(
539 server.since(cursor),
540 ) || undefined,
541 { tries: 120, delayMs: 500, server },
542 );
543 server.assertAlive("stack syntax error");
544
545 // EchoWorker is down (its bundle no longer builds), but every resource
546 // that does not depend on the broken module keeps serving: they live in
547 // the provider sidecar, which the exec child's crash does not touch.
548 expect((await fetchJson<{ marker: string }>(api("/marker"))).marker).toBe("api-v2");
549 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
550
551 // Recovery on the next save.
552 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v3"));
553 await waitForJson<{ marker: string }>(
554 "EchoWorker to recover on echo-v3",
555 echo("/marker"),
556 (body) => body.marker === "echo-v3",
557 { tries: 240, delayMs: 500, server },
558 );
559 await server.waitForPlanAfter(plansBefore);
560 cleanCursor = server.mark();
561 },
562 PHASE_TIMEOUT,
563 );
564
565 test(
566 "surviving a throw at module scope",
567 async () => {
568 const config = server.read("src/stack-config.ts");
569 server.write(
570 "src/stack-config.ts",
571 `${config}\nthrow new Error("dev-stress: deliberate module-scope failure");\n`,
572 );
573
574 await pollUntil(
575 "the CLI to report the failed run",
576 () =>
577 server.since(cleanCursor).includes("deliberate module-scope failure") ? true : undefined,
578 { tries: 120, delayMs: 500, server },
579 );
580 server.assertAlive("module-scope throw");
581 expect(server.since(cleanCursor)).toContain("alchemy dev: run failed");
582
583 // Restore and converge.
584 server.write("src/stack-config.ts", config);
585 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v4"));
586 await waitForJson<{ marker: string }>(
587 "EchoWorker to recover on echo-v4",
588 echo("/marker"),
589 (body) => body.marker === "echo-v4",
590 { tries: 240, delayMs: 500, server },
591 );
592 cleanCursor = server.mark();
593 },
594 PHASE_TIMEOUT,
595 );
596
597 test(
598 "surviving an apply failure: healthy resources keep serving while one resource cannot reconcile",
599 async () => {
600 // A second Worker that demands a port EchoWorker already holds. The
601 // stack imports and plans fine; the reconcile is what fails — which is
602 // the case that has to leave the rest of the stack serving. (A Worker
603 // with a missing `main` is NOT such a case: the local provider creates
604 // it happily.)
605 server.write("src/extra/squatter.ts", portSquatterSource);
606 server.patchRegion("alchemy.run.ts", "EXTRA", portSquatterDeclaration);
607
608 await pollUntil(
609 "the CLI to report the failed apply",
610 () => /alchemy dev: apply failed/.test(server.since(cleanCursor)) || undefined,
611 { tries: 240, delayMs: 500, server },
612 );
613 server.assertAlive("apply failure");
614 expect(server.since(cleanCursor)).toContain("already in use");
615
616 // Everything healthy is still healthy.
617 expect((await fetchJson<{ marker: string }>(echo("/marker"))).marker).toBe("echo-v4");
618 expect((await fetchJson<{ marker: string }>(api("/marker"))).marker).toBe("api-v2");
619 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe("hello from s3");
620
621 server.patchRegion("alchemy.run.ts", "EXTRA", "");
622 server.remove("src/extra/squatter.ts");
623 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v5"));
624 await waitForJson<{ marker: string }>(
625 "the stack to converge again",
626 echo("/marker"),
627 (body) => body.marker === "echo-v5",
628 { tries: 240, delayMs: 500, server },
629 );
630 cleanCursor = server.mark();
631 },
632 PHASE_TIMEOUT,
633 );
634
635 // ───────────────────────────────────────────────────────────────────────
636 // Phase 5 — the resource GRAPH changes shape. This is the substance of the
637 // suite: resources appear, get renamed, get replaced and disappear while
638 // the dev server keeps running, across both clouds and every resource kind
639 // the stack has (Workers, a Lambda, buckets, a queue + its event source, a
640 // Durable Object class, a MicroVM image).
641 // ───────────────────────────────────────────────────────────────────────
642
643 test(
644 "graph churn: a Cloudflare Worker can be added, renamed onto a new port, and removed",
645 async () => {
646 // ── add ──
647 server.write("src/extra/extra-worker.ts", extraWorkerSource("extra-v1"));
648 server.patchRegion("alchemy.run.ts", "EXTRA", extraWorkerDeclaration("ExtraWorker", "extra"));
649 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", extraWorkerOutput);
650 await waitForJson<{ marker: string }>(
651 "the added ExtraWorker to serve",
652 extra("/"),
653 (body) => body.marker === "extra-v1",
654 { tries: 240, delayMs: 500, server },
655 );
656 // The worker serves before the stack's re-apply prints its outputs —
657 // the output line is the proof the ENGINE saw the new resource.
658 await pollUntil("extraUrl in the stack outputs", () => server.outputUrl("extraUrl"), {
659 tries: 120,
660 delayMs: 500,
661 server,
662 });
663 server.assertAlive("worker added");
664
665 // ── rename (a new logical id is a create + a delete) ──
666 // The new generation lands on its own port: the engine may create
667 // before it deletes, and two `strictPort` workers cannot share one.
668 server.patchRegion(
669 "alchemy.run.ts",
670 "EXTRA",
671 extraWorkerDeclaration("RenamedWorker", "extraAlt"),
672 );
673 server.write("src/extra/extra-worker.ts", extraWorkerSource("extra-v2"));
674 await waitForJson<{ marker: string }>(
675 "the renamed worker to serve on its new port",
676 extraAlt("/"),
677 (body) => body.marker === "extra-v2",
678 { tries: 240, delayMs: 500, server },
679 );
680 // …and the old generation is gone.
681 await pollUntil(
682 "the old worker's port to stop serving",
683 async () => ((await isClosed(extra("/"))) ? true : undefined),
684 { tries: 120, delayMs: 500, server },
685 );
686 server.assertAlive("worker renamed");
687
688 // ── remove (output reference first, so the stack never names a
689 // binding that no longer exists) ──
690 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", "");
691 server.patchRegion("alchemy.run.ts", "EXTRA", "");
692 server.remove("src/extra/extra-worker.ts");
693 await pollUntil(
694 "the removed worker to stop serving",
695 async () => ((await stoppedServing(extraAlt("/"), "extra-v2")) ? true : undefined),
696 { tries: 240, delayMs: 500, server },
697 );
698 server.assertAlive("worker removed");
699
700 // The survivors are untouched.
701 expect((await fetchJson<{ marker: string }>(echo("/marker"))).marker).toBe("echo-v5");
702 cleanCursor = server.mark();
703 },
704 PHASE_TIMEOUT,
705 );
706
707 test(
708 "graph churn: a whole AWS Lambda (with its own S3 bucket) can be added and removed",
709 async () => {
710 // ── add: a new module, a new top-level import in the stack, a new
711 // Function resource, a new Bucket resource, and two new bindings ──
712 server.write("src/extra/ReportFunction.ts", reportFunctionSource("report-v1"));
713 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", reportFunctionImport);
714 server.patchRegion("alchemy.run.ts", "EXTRA", reportFunctionDeclaration);
715 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", reportFunctionOutput);
716
717 const reportUrl = await pollUntil(
718 "the new Lambda's function URL in the stack outputs",
719 () => server.outputUrl("reportUrl"),
720 { tries: 480, delayMs: 500, server },
721 );
722 // Local identity: a floci URL, never real AWS.
723 expect(reportUrl).toContain("localhost:4566");
724
725 const report = await fetchJson<{ marker: string; text: string }>(
726 new URL("/report", reportUrl),
727 undefined,
728 { tries: 240, delayMs: 500 },
729 );
730 expect(report).toEqual({ marker: "report-v1", text: "report-v1" });
731 server.assertAlive("lambda added");
732
733 // ── change it: a new marker means a new bundle for a Function that
734 // only came into existence a moment ago ──
735 server.write("src/extra/ReportFunction.ts", reportFunctionSource("report-v2"));
736 await waitForJson<{ marker: string }>(
737 "the added Lambda to hot-swap to report-v2",
738 new URL("/report", reportUrl),
739 (body) => body.marker === "report-v2",
740 { tries: 480, delayMs: 500, server },
741 );
742
743 // ── remove: the Function, its Bucket and its URL all go away ──
744 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", "");
745 server.patchRegion("alchemy.run.ts", "EXTRA", "");
746 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", "");
747 server.remove("src/extra/ReportFunction.ts");
748 await pollUntil(
749 "the removed Lambda to stop answering",
750 async () => {
751 try {
752 const res = await fetch(new URL("/report", reportUrl), {
753 signal: AbortSignal.timeout(5_000),
754 });
755 return res.ok ? undefined : true;
756 } catch {
757 return true;
758 }
759 },
760 { tries: 240, delayMs: 500, server },
761 );
762 server.assertAlive("lambda removed");
763
764 // The original Lambda is untouched by its sibling's whole lifecycle.
765 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe("hello from s3");
766 cleanCursor = server.mark();
767 },
768 PHASE_TIMEOUT,
769 );
770
771 test(
772 "graph churn: a Queue, its consumer event source, and a new Durable Object class are grafted onto a LIVE Worker",
773 async () => {
774 const plansBefore = server.planCount;
775
776 // Three kinds of change at once, all on a Worker that is already
777 // serving: a new resource (the Queue), a new event source (its
778 // consumer), and a new Durable Object class — which means a class
779 // migration on a running script.
780 server.patchRegion("src/EchoWorker.ts", "ECHO_IMPORTS", echoStreamImport);
781 server.patchRegion("src/EchoWorker.ts", "ECHO_BINDINGS", echoQueueBindings);
782 server.patchRegion("src/EchoWorker.ts", "ECHO_ROUTES", echoQueueRoutes);
783 server.patchRegion("src/EchoWorker.ts", "ECHO_LAYERS", echoQueueLayers);
784 server.patchRegion("src/EchoWorker.ts", "ECHO_CLASSES", echoInboxClass);
785
786 // The bundler hot-swaps the new script BEFORE the stack re-apply has
787 // registered the `Inbox` class and the queue consumer, so for a beat
788 // the Worker serves a bundle whose DO binding workerd rejects
789 // (`DurableObject 'Inbox' not found`). A message sent in that window
790 // has no consumer and is lost. Wait for the re-apply to land first;
791 // the self-healing is what's under test, not the window.
792 await server.waitForPlanAfter(plansBefore, { tries: 480 });
793 await pollUntil(
794 "the re-apply that registers the Queue, consumer and DO to finish",
795 () => (/\[EchoQueueConsumer\] created/.test(server.since(cleanCursor)) ? true : undefined),
796 { tries: 480, delayMs: 500, server },
797 );
798
799 const id = crypto.randomUUID();
800 await pollUntil(
801 "the new queue route to accept a message",
802 async () => {
803 try {
804 const res = await fetch(echo(`/queue/send?id=${id}`), {
805 signal: AbortSignal.timeout(5_000),
806 });
807 return res.ok ? true : undefined;
808 } catch {
809 return undefined;
810 }
811 },
812 { tries: 480, delayMs: 500, server },
813 );
814
815 // Produce → the local broker delivers → the new DO records it.
816 const received = await waitForJson<{ ids: string[] }>(
817 "the queue message to reach the new Durable Object",
818 echo("/queue/received"),
819 (body) => body.ids.includes(id),
820 { tries: 240, delayMs: 500, server },
821 );
822 expect(received.ids).toContain(id);
823 server.assertAlive("queue + consumer + DO added");
824
825 // The Worker's pre-existing bindings still work after the graft.
826 expect((await fetchJson<{ value: string }>(echo("/kv?key=graft"))).value).toBe("kv:graft");
827 expect((await fetchJson<{ count: number }>(echo("/counter"))).count).toBeGreaterThan(0);
828
829 // ── and back out again: the routes, the consumer, the Queue and the
830 // DO class all disappear from a running Worker ──
831 server.patchRegion("src/EchoWorker.ts", "ECHO_ROUTES", "");
832 server.patchRegion("src/EchoWorker.ts", "ECHO_BINDINGS", "");
833 server.patchRegion("src/EchoWorker.ts", "ECHO_LAYERS", "");
834 server.patchRegion("src/EchoWorker.ts", "ECHO_CLASSES", "");
835 server.patchRegion("src/EchoWorker.ts", "ECHO_IMPORTS", "");
836 await pollUntil(
837 "the queue route to disappear",
838 async () => {
839 try {
840 const res = await fetch(echo(`/queue/send?id=${id}`), {
841 signal: AbortSignal.timeout(5_000),
842 });
843 // The fall-through route answers with the marker instead.
844 if (!res.ok) return true;
845 const body = (await res.json()) as { sent?: string };
846 return body.sent === undefined ? true : undefined;
847 } catch {
848 return undefined;
849 }
850 },
851 { tries: 480, delayMs: 500, server },
852 );
853 server.assertAlive("queue + consumer + DO removed");
854 expect((await fetchJson<{ marker: string }>(echo("/marker"))).marker).toBe("echo-v5");
855 cleanCursor = server.mark();
856 },
857 PHASE_TIMEOUT,
858 );
859
860 test(
861 "graph churn: a second S3 bucket and its bindings are added to the LIVE Lambda, then removed",
862 async () => {
863 server.patchRegion("src/ApiFunction.ts", "LAMBDA_BINDINGS", lambdaArchiveBindings);
864 server.patchRegion("src/ApiFunction.ts", "LAMBDA_ROUTES", lambdaArchiveRoutes);
865
866 // Driven through the cross-cloud hop, so this also re-proves the
867 // Worker → Lambda edge after the Lambda's binding set changed.
868 const archived = await waitForJson<{ text: string }>(
869 "the new bucket binding to answer on the Lambda",
870 api("/aws/archive"),
871 (body) => body.text === "archived",
872 { tries: 480, delayMs: 500, server },
873 );
874 expect(archived.text).toBe("archived");
875 server.assertAlive("lambda bucket added");
876
877 // Pre-existing bindings survived the change.
878 expect((await fetchJson<{ text: string }>(api("/aws/dynamo"))).text).toBe("hello from dynamo");
879 // The engine just UPDATED the Function — hot swap must still work.
880 await expectLambdaHotSwapStillWorks("archive-added");
881
882 server.patchRegion("src/ApiFunction.ts", "LAMBDA_ROUTES", "");
883 server.patchRegion("src/ApiFunction.ts", "LAMBDA_BINDINGS", "");
884 await pollUntil(
885 "the archive route to disappear",
886 async () => {
887 try {
888 const res = await fetch(api("/aws/archive"), {
889 signal: AbortSignal.timeout(10_000),
890 });
891 return res.status === 404 ? true : undefined;
892 } catch {
893 return undefined;
894 }
895 },
896 { tries: 480, delayMs: 500, server },
897 );
898 server.assertAlive("lambda bucket removed");
899 await expectLambdaHotSwapStillWorks("archive-removed");
900 cleanCursor = server.mark();
901 },
902 PHASE_TIMEOUT,
903 );
904
905 test(
906 "ECS hot reload: context content, the Dockerfile itself, an env prop, and an inline-Dockerfile prop all roll the running containers",
907 async () => {
908 // ── 1. a file in the build context (the classic watch path) ──
909 server.write("site/ecs/index.html", "ecs-site-v2\n");
910 await waitForText("the ECS service to serve the edited context file", ecs("/"), "ecs-site-v2");
911 server.assertAlive("ecs context reload");
912
913 // ── 2. the Dockerfile ITSELF (also a file in the context, but the
914 // change only lands if the image actually rebuilds) ──
915 server.write(
916 "site/ecs/Dockerfile",
917 server
918 .read("site/ecs/Dockerfile")
919 .replace("BAKED_MARKER=dockerfile-v1", "BAKED_MARKER=dockerfile-v2"),
920 );
921 await waitForText(
922 "the ECS service to serve the rebuilt Dockerfile marker",
923 ecs("/baked.txt"),
924 "dockerfile-v2",
925 );
926 // The context file survived the rebuild.
927 await waitForText(
928 "the context file to still serve after the Dockerfile rebuild",
929 ecs("/"),
930 "ecs-site-v2",
931 );
932 server.assertAlive("ecs dockerfile reload");
933
934 // ── 3. an env PROP (no file event at all — the engine registers a new
935 // task-definition revision, and the running task must roll onto it;
936 // regression: only file-watch triggers restarted tasks, so prop-driven
937 // updates left containers serving the old revision forever) ──
938 server.patchRegion("alchemy.run.ts", "ECS_ENV", ' STRESS_ENV: "ecs-env-v2",\n');
939 await waitForText("the ECS task to roll onto the new env", ecs("/env.txt"), "ecs-env-v2");
940 server.assertAlive("ecs env-prop reload");
941
942 // ── 4. an INLINE Dockerfile (a pure prop change that is nonetheless a
943 // Dockerfile edit — same engine path as 3, but the image itself must
944 // rebuild) ──
945 server.patchRegion(
946 "alchemy.run.ts",
947 "ECS_INLINE_MARKER",
948 ` "RUN mkdir -p /www && echo -n ecs-inline-v2 > /www/index.html",\n`,
949 );
950 await waitForText(
951 "the inline-Dockerfile service to serve the rebuilt image",
952 ecsInline("/"),
953 "ecs-inline-v2",
954 );
955 server.assertAlive("ecs inline-dockerfile reload");
956
957 // The sibling service was untouched by the inline rebuild.
958 await waitForText("the context service to still serve", ecs("/"), "ecs-site-v2", { tries: 30 });
959 cleanCursor = server.mark();
960 },
961 PHASE_TIMEOUT,
962 );
963
964 test(
965 "EC2 hot reload: the hosted instance serves, and editing its program updates it in place",
966 async () => {
967 // The box's address comes from floci: `i-….localhost.floci.io`
968 // resolves to 127.0.0.1 and the mux publishes the SG app port.
969 const dns = await pollUntil("ec2Dns in the stack outputs", () => server.outputValue("ec2Dns"), {
970 tries: 240,
971 delayMs: 500,
972 server,
973 });
974 expect(dns).toMatch(/\.localhost\.floci\.io$/);
975 const marker = at2(dns, PORTS.ec2, "/marker");
976
977 // First boot: container start + userData (bundle sync from emulated
978 // S3, runtime install) + the Bun HTTP server binding the app port.
979 await waitForJson<{ marker: string }>(
980 "the EC2 box to serve its hosted program",
981 marker,
982 (body) => body.marker === "ec2-v1",
983 { tries: 90, delayMs: 1_000, server },
984 );
985
986 // The reload path is the ENGINE's: a content edit re-plans, the
987 // provider re-uploads the bundle in place and reboots the instance —
988 // same instance id, same address, new code.
989 const reloadStartedAt = Date.now();
990 server.write("src/ec2/marker.ts", markerModule("EC2_MARKER", "ec2-v2"));
991 await waitForJson<{ marker: string }>(
992 "the EC2 box to serve ec2-v2 after the in-place update",
993 marker,
994 (body) => body.marker === "ec2-v2",
995 { tries: 150, delayMs: 1_000, server },
996 );
997 console.log(`ec2 hosted-program reload -> serving ec2-v2 in ${Date.now() - reloadStartedAt}ms`);
998 // Same box, same address — the update was in place, not a replacement.
999 expect(server.outputValue("ec2Dns")).toBe(dns);
1000 server.assertAlive("ec2 hot reload");
1001 },
1002 PHASE_TIMEOUT,
1003 );
1004
1005 test(
1006 "Cloudflare Container hot reload: editing the container's program rebuilds the image and restarts it",
1007 async () => {
1008 // The container module is imported by the stack, so this edit replans;
1009 // the reload rides the image content hash in the worker's restart
1010 // config (regression: the sidecar-lifetime artifact memo made the diff
1011 // compare the first run's hash forever, and the config only carried
1012 // the image's stable paths — the running container served stale code
1013 // until a full dev-session restart).
1014 server.patchRegion(
1015 "src/SandboxContainer.ts",
1016 "SANDBOX_MARKER",
1017 ' marker: "sandbox-v2",\n',
1018 );
1019 await waitForJson<{ marker: string }>(
1020 "the sandbox container to serve the rebuilt program",
1021 echo("/sandbox"),
1022 (body) => body.marker === "sandbox-v2",
1023 { tries: 300, delayMs: 1_000, server },
1024 );
1025 server.assertAlive("container hot reload");
1026
1027 // The #1334 loopback rewrite still holds on the rebuilt container.
1028 const hostFetch = await fetchJson<{ body: string }>(echo("/sandbox/host-fetch"), undefined, {
1029 tries: 60,
1030 delayMs: 1_000,
1031 });
1032 expect(hostFetch.body).toContain("aws-site-env-v1");
1033 cleanCursor = server.mark();
1034 },
1035 PHASE_TIMEOUT,
1036 );
1037
1038 test(
1039 "graph churn: a replacement-forcing prop change swaps a DynamoDB table under a running Lambda",
1040 async () => {
1041 const source = server.read("src/ApiFunction.ts");
1042 // The partition key is immutable on DynamoDB — changing it is a
1043 // REPLACEMENT, not an update: a new table is created, the Lambda's
1044 // binding is repointed, and the old table is deleted.
1045 server.write(
1046 "src/ApiFunction.ts",
1047 source
1048 .replace('partitionKey: "id",', 'partitionKey: "pk",')
1049 .replace('attributes: { id: "S" },', 'attributes: { pk: "S" },')
1050 // every item key in the handler, in one sweep
1051 .replaceAll("id: { S:", "pk: { S:"),
1052 );
1053
1054 // The old table answers `/dynamo` until the replacement lands, so the
1055 // engine's own plan line is the signal — then the route must work
1056 // against the NEW table (a fresh write lands in it).
1057 const replacedCursor = server.mark();
1058 await pollUntil(
1059 "the engine to plan the table replacement",
1060 () => /\[StressTable\] replace/.test(server.plain(replacedCursor)) || undefined,
1061 { tries: 240, delayMs: 500, server },
1062 );
1063 // A replacement prints `creating replacement` → `created` and then
1064 // cleans up the old generation; the cleanup line is the end of it.
1065 await pollUntil(
1066 "the replaced table to finish",
1067 () =>
1068 /\[StressTable\] Replaced resource cleanup complete/.test(server.plain(replacedCursor)) ||
1069 undefined,
1070 { tries: 480, delayMs: 500, server },
1071 );
1072 await waitForJson<{ text: string }>(
1073 "the replaced table to serve through the Lambda",
1074 api("/aws/dynamo"),
1075 (body) => body.text === "hello from dynamo",
1076 { tries: 480, delayMs: 500, server },
1077 );
1078 server.assertAlive("table replaced");
1079 await expectLambdaHotSwapStillWorks("table-replaced");
1080
1081 // Put it back — the swap back is a second replacement.
1082 const restoredCursor = server.mark();
1083 server.write("src/ApiFunction.ts", source);
1084 await pollUntil(
1085 "the table's second replacement to finish",
1086 () =>
1087 /\[StressTable\] Replaced resource cleanup complete/.test(server.plain(restoredCursor)) ||
1088 undefined,
1089 { tries: 480, delayMs: 500, server },
1090 );
1091 await waitForJson<{ text: string }>(
1092 "the restored table to serve through the Lambda",
1093 api("/aws/dynamo"),
1094 (body) => body.text === "hello from dynamo",
1095 { tries: 480, delayMs: 500, server },
1096 );
1097 server.assertAlive("table restored");
1098 await expectLambdaHotSwapStillWorks("table-restored");
1099 cleanCursor = server.mark();
1100 },
1101 PHASE_TIMEOUT,
1102 );
1103
1104 test(
1105 "graph churn: a SECOND AWS MicroVM image is added, driven from the Worker, and removed",
1106 async () => {
1107 // A new image resource (floci builds it), a new top-level import in
1108 // both the stack and the Worker, two new MicroVM bindings on a running
1109 // Worker, and a route that boots and terminates an instance of it.
1110 server.write("src/extra/WorkerImage.ts", secondImageSource("worker-vm-v1"));
1111 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", secondImageImport);
1112 server.patchRegion("alchemy.run.ts", "EXTRA_LAYERS", secondImageLayer);
1113 server.patchRegion("src/MicrovmWorker.ts", "VM_IMPORTS", secondImageWorkerImport);
1114 server.patchRegion("src/MicrovmWorker.ts", "VM_BINDINGS", secondImageBindings);
1115 server.patchRegion("src/MicrovmWorker.ts", "VM_ROUTES", secondImageRoutes);
1116
1117 const booted = await waitForJson<{ microvmId: string }>(
1118 "the second MicroVM image to build and boot an instance",
1119 microvm("/second"),
1120 (body) => typeof body.microvmId === "string" && body.microvmId.length > 0,
1121 { tries: 900, delayMs: 1_000, server },
1122 );
1123 expect(booted.microvmId).toBeTruthy();
1124 server.assertAlive("second microvm added");
1125
1126 // The FIRST image still works — adding a sibling did not disturb it.
1127 expect(
1128 (
1129 await fetchJson<{ marker: string }>(microvm("/roundtrip?message=sibling"), undefined, {
1130 tries: 300,
1131 delayMs: 1_000,
1132 })
1133 ).marker,
1134 ).toBe("vm-v1");
1135
1136 // ── remove the second image and everything that referenced it ──
1137 server.patchRegion("src/MicrovmWorker.ts", "VM_ROUTES", "");
1138 server.patchRegion("src/MicrovmWorker.ts", "VM_BINDINGS", "");
1139 server.patchRegion("src/MicrovmWorker.ts", "VM_IMPORTS", "");
1140 server.patchRegion("alchemy.run.ts", "EXTRA_LAYERS", "");
1141 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", "");
1142 server.remove("src/extra/WorkerImage.ts");
1143 await pollUntil(
1144 "the second MicroVM's route to disappear",
1145 async () => {
1146 try {
1147 const res = await fetch(microvm("/second"), {
1148 signal: AbortSignal.timeout(10_000),
1149 });
1150 const text = await res.text();
1151 return text === "ok" ? true : undefined;
1152 } catch {
1153 return undefined;
1154 }
1155 },
1156 { tries: 600, delayMs: 500, server },
1157 );
1158 server.assertAlive("second microvm removed");
1159 cleanCursor = server.mark();
1160 },
1161 PHASE_TIMEOUT,
1162 );
1163
1164 test(
1165 "graph churn: the entire AWS Lambda subsystem is deleted and restored while Cloudflare keeps serving",
1166 async () => {
1167 const awsHalf = server.read("alchemy.run.ts");
1168
1169 // Delete the Lambda, its bucket, its table, its queue and its event
1170 // source in one edit — and the cross-cloud binding that referenced it.
1171 server.patchRegion("alchemy.run.ts", "AWS_OUTPUTS", "");
1172 server.patchRegion("alchemy.run.ts", "AWS_LAMBDA_URL", "");
1173 server.patchRegion("alchemy.run.ts", "AWS_HALF", "");
1174
1175 await pollUntil(
1176 "the AWS half to be gone from ApiWorker's bindings",
1177 async () => {
1178 try {
1179 const body = (await (
1180 await fetch(api("/env"), { signal: AbortSignal.timeout(5_000) })
1181 ).json()) as { AWS_LAMBDA_URL: string | null };
1182 return body.AWS_LAMBDA_URL === null ? true : undefined;
1183 } catch {
1184 return undefined;
1185 }
1186 },
1187 { tries: 600, delayMs: 500, server },
1188 );
1189 server.assertAlive("aws half deleted");
1190
1191 // Cloudflare is completely unaffected by the AWS half vanishing.
1192 expect((await fetchJson<{ marker: string }>(echo("/marker"))).marker).toBe("echo-v5");
1193 expect((await fetchJson<{ text: string }>(echo("/r2"))).text).toBe("hello from r2");
1194 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
1195
1196 // ── restore the whole subsystem ──
1197 server.write("alchemy.run.ts", awsHalf);
1198 await waitForJson<{ marker: string }>(
1199 "the restored Lambda to serve through the cross-cloud hop",
1200 api("/aws/"),
1201 (body) => body.marker.startsWith("lambda-"),
1202 { tries: 900, delayMs: 500, server },
1203 );
1204 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe("hello from s3");
1205 server.assertAlive("aws half restored");
1206 // A brand-new Function generation must get a working watch loop.
1207 await expectLambdaHotSwapStillWorks("aws-half-restored");
1208 cleanCursor = server.mark();
1209 },
1210 PHASE_TIMEOUT,
1211 );
1212
1213 test(
1214 "binding churn: changing a binding value re-applies the consumer; changing a dev child's env restarts it",
1215 async () => {
1216 // A plain var binding on the path-`main` Worker.
1217 server.patchRegion(
1218 "alchemy.run.ts",
1219 "API_VARIABLE",
1220 ' API_VARIABLE: "api-variable-v2",\n',
1221 );
1222 await waitForJson<{ API_VARIABLE: string }>(
1223 "ApiWorker to pick up the new binding value",
1224 api("/env"),
1225 (body) => body.API_VARIABLE === "api-variable-v2",
1226 { tries: 240, delayMs: 500, server },
1227 );
1228
1229 // A brand-new binding on a Worker that is already running.
1230 server.patchRegion("alchemy.run.ts", "API_EXTRA_ENV", ' API_EXTRA: "api-extra-v1",\n');
1231 await waitForJson<{ API_EXTRA: string | null }>(
1232 "ApiWorker to gain a binding it never had",
1233 api("/env"),
1234 (body) => body.API_EXTRA === "api-extra-v1",
1235 { tries: 240, delayMs: 500, server },
1236 );
1237
1238 // …and lose it again.
1239 server.patchRegion("alchemy.run.ts", "API_EXTRA_ENV", "");
1240 await waitForJson<{ API_EXTRA: string | null }>(
1241 "ApiWorker to lose the binding again",
1242 api("/env"),
1243 (body) => body.API_EXTRA === null,
1244 { tries: 240, delayMs: 500, server },
1245 );
1246
1247 // `Command.Dev`'s restart surface: a change to the resolved config must
1248 // restart the child (new pid), not silently leave the old one running.
1249 const pidBefore = (await fetchJson<{ pid: number }>(awsSite("/__dev-env"))).pid;
1250 server.patchRegion(
1251 "alchemy.run.ts",
1252 "SITE_MARKER",
1253 ' SITE_MARKER: "aws-site-env-v2",\n',
1254 );
1255 const restarted = await waitForJson<{ marker: string; pid: number }>(
1256 "the dev-command child to restart with the new env",
1257 awsSite("/__dev-env"),
1258 (body) => body.marker === "aws-site-env-v2",
1259 { tries: 240, delayMs: 500, server },
1260 );
1261 expect(restarted.pid).not.toBe(pidBefore);
1262 server.assertAlive("binding churn");
1263 await expectLambdaHotSwapStillWorks("binding-churn");
1264 cleanCursor = server.mark();
1265 },
1266 PHASE_TIMEOUT,
1267 );
1268
1269 // ───────────────────────────────────────────────────────────────────────
1270 // Phase 6 — rapid-fire edits. Convergence to the LAST write, and evidence
1271 // that the watchers coalesce instead of replaying every keystroke.
1272 // ───────────────────────────────────────────────────────────────────────
1273
1274 test(
1275 "rapid-fire bundler edits converge on the last write without re-running the stack",
1276 async () => {
1277 // The previous phase edited stack-graph files (alchemy.run.ts, the
1278 // Lambda source); absorb any trailing re-plan before baselining.
1279 await server.settlePlans();
1280 const plansBefore = server.planCount;
1281 const BURST = 25;
1282 for (let i = 1; i <= BURST; i++) {
1283 server.write("src/api/marker.ts", markerModule("API_MARKER", `api-storm-${i}`));
1284 await Bun.sleep(40);
1285 }
1286
1287 await waitForJson<{ marker: string }>(
1288 `ApiWorker to settle on api-storm-${BURST}`,
1289 api("/marker"),
1290 (body) => body.marker === `api-storm-${BURST}`,
1291 { tries: 240, delayMs: 500, server },
1292 );
1293 // A burst on the bundler path must never escalate into stack re-runs.
1294 expect(server.planCount).toBe(plansBefore);
1295 server.assertAlive("bundler burst");
1296 },
1297 PHASE_TIMEOUT,
1298 );
1299
1300 test(
1301 "rapid-fire stack edits converge on the last write and are coalesced",
1302 async () => {
1303 const plansBefore = server.planCount;
1304 const BURST = 15;
1305 for (let i = 1; i <= BURST; i++) {
1306 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", `echo-storm-${i}`));
1307 await Bun.sleep(60);
1308 }
1309
1310 await waitForJson<{ marker: string }>(
1311 `EchoWorker to settle on echo-storm-${BURST}`,
1312 echo("/marker"),
1313 (body) => body.marker === `echo-storm-${BURST}`,
1314 { tries: 360, delayMs: 500, server },
1315 );
1316
1317 // The hot swap lands before the stack re-runs; wait for the re-plan
1318 // and let any coalesced follow-up runs drain before counting.
1319 await server.waitForPlanAfter(plansBefore);
1320 await Bun.sleep(15_000);
1321 const restarts = server.planCount - plansBefore;
1322 expect(restarts).toBeGreaterThanOrEqual(1);
1323 // Debouncing: a sub-second burst of 15 saves must not produce 15
1324 // complete plan/apply cycles.
1325 expect(restarts).toBeLessThan(BURST);
1326 server.assertAlive("stack burst");
1327 },
1328 PHASE_TIMEOUT,
1329 );
1330
1331 test(
1332 "simultaneous cross-cloud edits all land",
1333 async () => {
1334 // One tick, four surfaces: the bundler path, the watch path, the AWS
1335 // half, and the stack graph itself.
1336 server.write("src/api/marker.ts", markerModule("API_MARKER", "api-final"));
1337 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-final"));
1338 server.write("src/lambda/marker.ts", markerModule("LAMBDA_MARKER", "lambda-final"));
1339 server.patchRegion(
1340 "alchemy.run.ts",
1341 "API_VARIABLE",
1342 ' API_VARIABLE: "api-variable-final",\n',
1343 );
1344
1345 await waitForJson<{ marker: string }>(
1346 "EchoWorker to serve echo-final",
1347 echo("/marker"),
1348 (body) => body.marker === "echo-final",
1349 { tries: 360, delayMs: 500, server },
1350 );
1351 await waitForJson<{ marker: string }>(
1352 "ApiWorker to serve api-final",
1353 api("/marker"),
1354 (body) => body.marker === "api-final",
1355 { tries: 360, delayMs: 500, server },
1356 );
1357 await waitForJson<{ API_VARIABLE: string }>(
1358 "ApiWorker to carry api-variable-final",
1359 api("/env"),
1360 (body) => body.API_VARIABLE === "api-variable-final",
1361 { tries: 360, delayMs: 500, server },
1362 );
1363 await waitForJson<{ marker: string }>(
1364 "the Lambda to serve lambda-final",
1365 api("/aws/"),
1366 (body) => body.marker === "lambda-final",
1367 { tries: 360, delayMs: 1_000, server },
1368 );
1369 server.assertAlive("simultaneous edits");
1370 },
1371 PHASE_TIMEOUT,
1372 );
1373
1374 // ───────────────────────────────────────────────────────────────────────
1375 // Phase 7 — after all that churn, the whole stack is still the stack.
1376 // ───────────────────────────────────────────────────────────────────────
1377
1378 test(
1379 "after the full churn every resource still serves and the CLI never restarted",
1380 async () => {
1381 server.assertAlive("final health check");
1382
1383 // Cloudflare
1384 expect((await fetchJson<{ value: string }>(echo("/kv?key=final"))).value).toBe("kv:final");
1385 expect((await fetchJson<{ text: string }>(echo("/r2"))).text).toBe("hello from r2");
1386 expect((await fetchJson<{ count: number }>(echo("/counter"))).count).toBeGreaterThan(0);
1387 const finalSandbox = await fetchJson<{ greeting: string; marker: string }>(
1388 echo("/sandbox"),
1389 undefined,
1390 { tries: 60, delayMs: 1_000 },
1391 );
1392 expect(finalSandbox.greeting).toBe("hello-from-container");
1393 expect(finalSandbox.marker).toBe("sandbox-v2");
1394
1395 // AWS, through the cross-cloud hop
1396 expect((await fetchJson<{ text: string }>(api("/aws/dynamo"))).text).toBe("hello from dynamo");
1397 const message = { id: crypto.randomUUID() };
1398 await fetchOk(api("/aws/queue/send"), {
1399 method: "POST",
1400 body: JSON.stringify(message),
1401 });
1402 const delivered = await waitForJson<{ body: string | null }>(
1403 "the post-churn queue message to be consumed",
1404 api(`/aws/queue/messages?id=${message.id}`),
1405 (body) => body.body !== null,
1406 { tries: 180, delayMs: 500, server },
1407 );
1408 expect(JSON.parse(delivered.body!)).toEqual(message);
1409
1410 // ECS: both services still serve after every reload they went through.
1411 await waitForText("the ECS service post-churn", ecs("/"), "ecs-site-v2", {
1412 tries: 60,
1413 });
1414 await waitForText("the inline ECS service post-churn", ecsInline("/"), "ecs-inline-v2", {
1415 tries: 60,
1416 });
1417
1418 // Websites. The Cloudflare site's port is not pinned (a
1419 // `Website.StaticSite` owns its Worker's dev options), so re-read the
1420 // newest value the CLI printed rather than trusting the boot one.
1421 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
1422 cfSiteUrl = server.outputUrl("cfSiteUrl") ?? cfSiteUrl;
1423 expect(await (await fetchOk(new URL("/", cfSiteUrl))).text()).toContain("cf-site-v1");
1424
1425 // No failure was logged since the last phase that expected one.
1426 expect(server.since(cleanCursor)).not.toContain("alchemy dev: run failed");
1427 expect(server.since(cleanCursor)).not.toContain("alchemy dev: apply failed");
1428 },
1429 PHASE_TIMEOUT,
1430 );
1431
1432 test(
1433 "the MicroVM image survives every unrelated edit, and rebuilds when its own source changes",
1434 async () => {
1435 // Nothing the suite did to the other resources touched the image.
1436 expect(server.output).not.toContain("[StressMicrovm] replace");
1437 expect(
1438 (
1439 await fetchJson<{ marker: string }>(microvm("/roundtrip?message=post-churn"), undefined, {
1440 tries: 240,
1441 delayMs: 1_000,
1442 })
1443 ).marker,
1444 ).toBe("vm-v1");
1445
1446 // Now change the program that runs INSIDE the VM. The dev provider
1447 // builds the new image with a HOST-side cached `docker build` and hands
1448 // floci a pre-built `docker://` reference, so the rebuild itself is
1449 // seconds — the wait below is dominated by the roundtrip's own
1450 // boot-a-VM-per-request cost.
1451 const rebuildStartedAt = Date.now();
1452 server.write("src/vm/marker.ts", markerModule("VM_MARKER", "vm-v2"));
1453 await waitForJson<{ marker: string }>(
1454 "the rebuilt MicroVM image to serve vm-v2",
1455 microvm("/roundtrip?message=rebuild"),
1456 (body) => body.marker === "vm-v2",
1457 { tries: 600, delayMs: 1_000, server },
1458 );
1459 const rebuildMs = Date.now() - rebuildStartedAt;
1460 console.log(`microvm image rebuild -> serving vm-v2 in ${rebuildMs}ms`);
1461 // Pre-docker:// this took minutes (zip upload + cold server-side
1462 // build); the cached host build must keep the whole edit-to-serving
1463 // path within a couple of VM boots.
1464 expect(rebuildMs).toBeLessThan(120_000);
1465 server.assertAlive("microvm image rebuild");
1466 },
1467 PHASE_TIMEOUT,
1468 );
1469
1470 test(
1471 "shuts down cleanly on Ctrl-C and stops serving",
1472 async () => {
1473 await server.shutdown();
1474 expect(server.alive).toBe(false);
1475
1476 for (const url of [echo("/marker"), api("/marker"), awsSite("/")]) {
1477 await pollUntil(
1478 `${url} to stop serving after shutdown`,
1479 async () => ((await isClosed(url)) ? true : undefined),
1480 { tries: 60, delayMs: 500 },
1481 );
1482 }
1483 },
1484 PHASE_TIMEOUT,
1485 );