@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 9,884 lines 431 KB
Raw
1 import { randomBytes, createHash } from "node:crypto";
2 import * as Docker from "@alchemy.run/docker";
3 import * as Fly from "@alchemy.run/fly";
4 import { AppDeletionAmbiguous } from "@alchemy.run/fly/App";
5 import { readinessRoles, DeploymentRecoveryAmbiguous } from "@alchemy.run/fly/bluegreen";
6 import { makeMachineLeases } from "@alchemy.run/fly/leases";
7 import { type MachineContainer, type MachineProps } from "@alchemy.run/fly/Machine";
8 import { alchemyMetadataKeys as keys } from "@alchemy.run/fly/Metadata";
9 import {
10 waitHealthy,
11 autostopMode,
12 checksPassing,
13 ensureStarted,
14 observeReplicaSet,
15 predecessorShutdown,
16 deleteReplicaSet,
17 ReplicaNotCreated,
18 ReplicaRetirementIncomplete,
19 retireMachines,
20 retireMachine,
21 } from "@alchemy.run/fly/replicas";
22 import { ReplicaChecksNotPassing } from "@alchemy.run/fly/replicas";
23 import { expect, assert, it, describe } from "@alchemy.run/test";
24 import * as Test from "@alchemy.run/test/Alchemy";
25 import type { ScratchStack } from "@alchemy.run/test/Alchemy";
26 import * as TestCore from "@alchemy.run/test/Core";
27 import { scratchStack, withProviders } from "@alchemy.run/test/Core";
28 import {
29 GatewayTimeout,
30 HTTP_STATUS_MAP,
31 RETRYABLE_HTTP_STATUSES,
32 } from "@distilled.cloud/core/errors";
33 import { Credentials } from "@distilled.cloud/fly-io/Credentials";
34 import { GatewayTimeout as GatewayTimeoutErrors } from "@distilled.cloud/fly-io/Errors";
35 import * as machines from "@distilled.cloud/fly-io/machines";
36 import { type Machine } from "@distilled.cloud/fly-io/machines";
37 import * as Retry from "@distilled.cloud/fly-io/Retry";
38 import * as Alchemy from "alchemy";
39 import { DestroyError } from "alchemy/Apply";
40 import { localState, makeLocalState } from "alchemy/State/LocalState";
41 import * as Cause from "effect/Cause";
42 import * as Clock from "effect/Clock";
43 import * as ConfigProvider from "effect/ConfigProvider";
44 import * as Data from "effect/Data";
45 import * as Deferred from "effect/Deferred";
46 import * as Effect from "effect/Effect";
47 import * as Exit from "effect/Exit";
48 import * as Fiber from "effect/Fiber";
49 import * as FileSystem from "effect/FileSystem";
50 import * as FetchHttpClient from "effect/http/FetchHttpClient";
51 import * as HttpClient from "effect/http/HttpClient";
52 import * as HttpClientRequest from "effect/http/HttpClientRequest";
53 import * as Path from "effect/Path";
54 import * as Redacted from "effect/Redacted";
55 import * as Ref from "effect/Ref";
56 import * as Result from "effect/Result";
57 import * as Schedule from "effect/Schedule";
58 import * as Schema from "effect/Schema";
59 import * as Stream from "effect/Stream";
60 import { engineActor } from "./fixtures/actors.ts";
61 import { dropCompletedCreate } from "./fixtures/bluegreen-create-proxy.ts";
62 import {
63 Site,
64 Token,
65 TRIGGER_SECRET,
66 Writer,
67 writerLayer,
68 } from "./fixtures/bluegreen-runtime-secrets/writer.ts";
69 import {
70 BoundSecrets,
71 CacheOne,
72 CacheTwo,
73 Site as SiteBluegreenSecrets,
74 } from "./fixtures/bluegreen-secrets.ts";
75 import {
76 assertOrder,
77 assertReplacement,
78 assertStopped,
79 makeScenario,
80 requireValue,
81 } from "./fixtures/bluegreen-worker-test.ts";
82 import {
83 assertAppGone,
84 census,
85 checks,
86 deployWorker,
87 assertCommitted,
88 } from "./fixtures/bluegreen.ts";
89 import { sanitizeExecFailure } from "./fixtures/exec-lease.ts";
90 import { makeReadinessControl, repairReadiness } from "./fixtures/http-readiness-control.ts";
91 import {
92 assertReadinessCommit,
93 readinessActor,
94 readinessChecksPassing,
95 readinessProxy,
96 retires,
97 type ReadinessEvent,
98 } from "./fixtures/idle-cadence-readiness.ts";
99 import {
100 observeStops,
101 type StopRequest,
102 writeLegacyProtocol,
103 } from "./fixtures/legacy-protocol-writer.ts";
104 import MountedBlueGreen from "./fixtures/mounted-bluegreen.ts";
105 import {
106 assertBarrierInventory,
107 assertClean,
108 assertConverged,
109 assertSingleRunner,
110 census as censusProcessDeath,
111 deploy,
112 evidencePaths,
113 heldLeases,
114 identity,
115 machine,
116 matchesBarrier,
117 nowSeconds,
118 observeLeaseExpiry,
119 persistedRow,
120 phases,
121 processDeathFile,
122 readWitness,
123 writeEvidence,
124 type Phase,
125 type Witness,
126 } from "./fixtures/process-death.ts";
127 import {
128 appName,
129 candidateId,
130 MachineServiceAutostop,
131 metadata,
132 protocolClient,
133 reconcile,
134 reply,
135 withControlledClient,
136 } from "./fixtures/protocol-branches.ts";
137 import {
138 Finalized,
139 RunnerInterrupted,
140 Witness as WitnessSignalOverlap,
141 assertBoundary,
142 assertConverged as assertConvergedSignalOverlap,
143 assertInventory,
144 assertReleased,
145 assertSingleRunner as assertSingleRunnerSignalOverlap,
146 boundary,
147 cases,
148 firstReturnedUncordon,
149 matches,
150 observeExpiry,
151 observeLeases,
152 pathsFor,
153 readEvidence,
154 signalOverlapFile,
155 successful,
156 type SignalCase,
157 } from "./fixtures/signal-overlap.ts";
158 import { delayedResourceWrite, ResourceRow } from "./fixtures/state-persistence.ts";
159 import { transportProxy, throughProxy, type TransportEvent } from "./fixtures/transport.ts";
160
161 // Blocks are independent (each case deploys its own stack). Fully parallel
162 // overloads Fly's API (slow App deletes, missed timing windows), so they run
163 // in six concurrent shards balanced by measured duration (~35 min each);
164 // blocks run one at a time within a shard, and process-wide cases keep
165 // `exclusive: true`.
166 /** URL param values may be `Redacted` (sent unwrapped); compare their plain text. */
167 const unredact = (value: string | Redacted.Redacted | undefined) =>
168 Redacted.isRedacted(value) ? String(Redacted.value(value)) : value;
169
170 const decodeAutostop = Schema.decodeUnknownSync(MachineServiceAutostop);
171
172 describe.concurrent("BlueGreen", () => {
173 describe.sequential("shard 1", () => {
174 describe.sequential(
175 "idle topology",
176 {
177 tags: [
178 "provider:fly",
179 "provider:fly:app",
180 "provider:fly:machine",
181 "provider:fly:service",
182 "live",
183 ],
184 },
185 () => {
186 const file = "test/Fly/BlueGreen.test.ts";
187 const { test } = Test.make({ providers: Fly.providers() });
188 type IdleMode = "stop" | "suspend";
189
190 const topology = (
191 mode: IdleMode,
192 count: number,
193 floor = 0,
194 mixed = false,
195 ): Partial<Omit<Extract<MachineProps, { image: string }>, "app">> => ({
196 count,
197 deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
198 services: [
199 {
200 protocol: "tcp",
201 internalPort: 80,
202 autostop: mode,
203 autostart: true,
204 minMachinesRunning: floor,
205 checks: [checks.ready],
206 },
207 ...(mixed ? [{ protocol: "tcp", internalPort: 81, autostop: "off" as const }] : []),
208 ],
209 });
210
211 const waitState = (appName: string, machineId: string, state: string) =>
212 machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
213 Effect.repeat({
214 schedule: Schedule.spaced("2 seconds"),
215 times: 30,
216 until: (machine) => machine.state === state,
217 }),
218 Effect.tap((machine) => Effect.sync(() => expect(machine.state).toBe(state))),
219 Effect.timeout("90 seconds"),
220 );
221
222 // Native stop/suspend is setup only; automatic return-to-idle has its own test.
223 const idleAll = (appName: string, ids: string[], mode: IdleMode) =>
224 Effect.forEach(ids, (id) =>
225 Effect.gen(function* () {
226 const target = { app_name: appName, machine_id: id };
227 const current = yield* machines.getMachine(target);
228 if (current.state !== "started") {
229 yield* machines.startMachine(target);
230 yield* waitState(appName, id, "started");
231 }
232 if (mode === "stop") {
233 yield* machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" });
234 } else {
235 yield* machines.suspendMachine(target);
236 }
237 yield* waitState(appName, id, mode === "stop" ? "stopped" : "suspended");
238 }),
239 );
240
241 // Expected slots are literal scenario inputs, never computed by readinessRoles.
242 const assertSlots = (live: machines.Machine[], runningSlots: readonly number[]) => {
243 const ordered = [...live].sort(
244 (left, right) =>
245 Number(left.config?.metadata?.["alchemy.replica"]) -
246 Number(right.config?.metadata?.["alchemy.replica"]),
247 );
248 expect(ordered.map((machine) => machine.config?.metadata?.["alchemy.replica"])).toEqual(
249 Array.from({ length: live.length }, (_, index) => String(index)),
250 );
251 expect(
252 ordered
253 .filter((machine) => machine.config?.metadata?.["alchemy.readiness-role"] === "run")
254 .map((machine) => Number(machine.config?.metadata?.["alchemy.replica"])),
255 ).toEqual([...runningSlots]);
256 const roles = Array.from({ length: live.length }, (_, slot) =>
257 runningSlots.includes(slot) ? "run" : "idle",
258 ).join(",");
259 for (const [slot, machine] of ordered.entries()) {
260 expect(machine.config?.metadata?.["alchemy.readiness-role"]).toBe(
261 runningSlots.includes(slot) ? "run" : "idle",
262 );
263 expect(machine.config?.metadata?.["alchemy.readiness-roles"]).toBe(roles);
264 }
265 return ordered;
266 };
267
268 const assertTopology = (
269 appName: string,
270 ids: string[],
271 mode: IdleMode,
272 runningSlots: readonly number[],
273 floor = 0,
274 mixed = false,
275 ) =>
276 Effect.gen(function* () {
277 const live = yield* assertCommitted(appName, ids);
278 const ordered = assertSlots(live, runningSlots);
279 expect(
280 new Set(live.map((machine) => machine.config?.metadata?.["alchemy.generation"])).size,
281 ).toBe(1);
282 for (const [slot, machine] of ordered.entries()) {
283 expect(machine.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
284 expect(machine.cordoned).toBe(false);
285 expect(machine.config?.services).toHaveLength(mixed ? 2 : 1);
286 expect(autostopMode(machine.config?.services?.[0]?.autostop)).toBe(mode);
287 expect(machine.config?.services?.[0]?.autostart).toBe(true);
288 expect(machine.config?.services?.[0]?.min_machines_running).toBe(floor);
289 if (mixed) {
290 expect(autostopMode(machine.config?.services?.[1]?.autostop)).toBe("off");
291 }
292 if (!runningSlots.includes(slot)) {
293 expect(["created", "stopped", "suspended"]).toContain(machine.state);
294 } else {
295 expect(mixed ? ["started"] : ["started", "stopped", "suspended"]).toContain(
296 machine.state,
297 );
298 expect(machine.instance_id).toBeDefined();
299 expect(machine.config?.metadata?.["alchemy.checked-instance"]).toBe(
300 machine.instance_id,
301 );
302 }
303 }
304 return ordered;
305 });
306
307 const checkedDeployment = (
308 actor: Test.ScratchStack,
309 proxy: { events: TransportEvent[]; readiness: ReadinessEvent[] },
310 version: string,
311 mode: IdleMode,
312 count: number,
313 runningSlots: readonly number[],
314 priorIds: readonly string[] = [],
315 floor = 0,
316 mixed = false,
317 ) =>
318 Effect.gen(function* () {
319 const begin = proxy.events.length;
320 const proofBegin = proxy.readiness.length;
321 const output = yield* deployWorker(
322 actor,
323 version,
324 topology(mode, count, floor, mixed),
325 ).pipe(Effect.scoped);
326 const live = yield* assertTopology(
327 output.appName,
328 output.machineIds,
329 mode,
330 runningSlots,
331 floor,
332 mixed,
333 );
334 assertReadinessCommit(
335 proxy.readiness.slice(proofBegin),
336 priorIds,
337 live,
338 runningSlots,
339 ["ready", "servicecheck-00-http-80"],
340 !mixed,
341 );
342 expect(
343 proxy.events
344 .slice(begin)
345 .some(
346 (event) =>
347 event.stage === "request" &&
348 priorIds.includes(event.machineId!) &&
349 event.path.endsWith("/start"),
350 ),
351 ).toBe(false);
352 return output;
353 });
354
355 // Each case deploys its own stack; these are the file's longest cases
356 // (3-8 min of real idle waits), so run them side by side.
357 describe.concurrent("live idle topology and automatic cadence", () => {
358 for (const mode of ["stop", "suspend"] as const) {
359 test.provider(
360 `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
361 (stack) =>
362 Effect.gen(function* () {
363 yield* stack.destroy();
364 const site = yield* stack.deploy(Fly.App("Site"));
365 try {
366 const proxy = yield* readinessProxy();
367 const actor = yield* readinessActor(
368 stack,
369 `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
370 file,
371 proxy,
372 );
373 const first = yield* checkedDeployment(actor, proxy, "one", mode, 1, [0]);
374 yield* idleAll(site.appName, first.machineIds, mode);
375 const up = yield* checkedDeployment(
376 actor,
377 proxy,
378 "two",
379 mode,
380 3,
381 [0],
382 first.machineIds,
383 );
384 expect(up.machineIds).toHaveLength(3);
385 expect(up.machineIds.every((id) => !first.machineIds.includes(id))).toBe(true);
386 yield* idleAll(site.appName, up.machineIds, mode);
387 const down = yield* checkedDeployment(
388 actor,
389 proxy,
390 "three",
391 mode,
392 1,
393 [0],
394 up.machineIds,
395 );
396 expect(down.machineIds).toHaveLength(1);
397 expect(up.machineIds).not.toContain(down.machineId);
398 yield* idleAll(site.appName, down.machineIds, mode);
399 const unchanged = yield* deployWorker(actor, "three", topology(mode, 1));
400 expect(unchanged.machineIds).toEqual(down.machineIds);
401 expect((yield* census(site.appName))[0]!.state).toBe(
402 mode === "stop" ? "stopped" : "suspended",
403 );
404 } finally {
405 yield* stack.destroy();
406 yield* assertAppGone(site.appName);
407 }
408 }).pipe(Effect.scoped),
409 { timeout: 900_000 },
410 );
411
412 test.provider(
413 `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
414 (stack) =>
415 Effect.gen(function* () {
416 yield* stack.destroy();
417 const site = yield* stack.deploy(Fly.App("Site"));
418 try {
419 const proxy = yield* readinessProxy();
420 const actor = yield* readinessActor(
421 stack,
422 `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
423 file,
424 proxy,
425 );
426 const first = yield* checkedDeployment(actor, proxy, "one", mode, 3, [0]);
427 yield* idleAll(site.appName, first.machineIds, mode);
428 const raised = yield* checkedDeployment(
429 actor,
430 proxy,
431 "one",
432 mode,
433 3,
434 [0, 1],
435 first.machineIds,
436 2,
437 );
438 yield* idleAll(site.appName, raised.machineIds, mode);
439 const lowered = yield* checkedDeployment(
440 actor,
441 proxy,
442 "one",
443 mode,
444 3,
445 [0],
446 raised.machineIds,
447 );
448 yield* idleAll(site.appName, lowered.machineIds, mode);
449 const slots = assertSlots(yield* census(site.appName), [0]);
450 yield* machines.startMachine({
451 app_name: site.appName,
452 machine_id: slots[2]!.id!,
453 });
454 yield* waitState(site.appName, slots[2]!.id!, "started");
455 const mixedOld = (yield* census(site.appName)).sort(
456 (a, b) =>
457 Number(a.config?.metadata?.["alchemy.replica"]) -
458 Number(b.config?.metadata?.["alchemy.replica"]),
459 );
460 expect(mixedOld.map((machine) => machine.state)).toEqual([
461 mode === "stop" ? "stopped" : "suspended",
462 mode === "stop" ? "stopped" : "suspended",
463 "started",
464 ]);
465 const retained = yield* checkedDeployment(
466 actor,
467 proxy,
468 "retained",
469 mode,
470 3,
471 [2],
472 lowered.machineIds,
473 );
474 yield* idleAll(site.appName, retained.machineIds, mode);
475 const retainedSlots = assertSlots(yield* census(site.appName), [2]);
476 yield* machines.startMachine({
477 app_name: site.appName,
478 machine_id: retainedSlots[2]!.id!,
479 });
480 yield* waitState(site.appName, retainedSlots[2]!.id!, "started");
481 expect(
482 assertSlots(yield* census(site.appName), [2]).map((machine) => machine.state),
483 ).toEqual([
484 mode === "stop" ? "stopped" : "suspended",
485 mode === "stop" ? "stopped" : "suspended",
486 "started",
487 ]);
488 const retainedFloor = yield* checkedDeployment(
489 actor,
490 proxy,
491 "retained",
492 mode,
493 3,
494 [0, 2],
495 retained.machineIds,
496 2,
497 );
498 yield* idleAll(site.appName, retainedFloor.machineIds, mode);
499 const mixed = yield* checkedDeployment(
500 actor,
501 proxy,
502 "one",
503 mode,
504 3,
505 [0, 1, 2],
506 retainedFloor.machineIds,
507 0,
508 true,
509 );
510 yield* idleAll(site.appName, mixed.machineIds, mode);
511 yield* checkedDeployment(actor, proxy, "one", mode, 3, [0], mixed.machineIds);
512 } finally {
513 yield* stack.destroy();
514 yield* assertAppGone(site.appName);
515 }
516 }).pipe(Effect.scoped),
517 { timeout: 1_200_000 },
518 );
519
520 for (const recoveredCount of [3, 1, 4]) {
521 test.provider(
522 `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
523 (stack) =>
524 Effect.gen(function* () {
525 yield* stack.destroy();
526 const site = yield* stack.deploy(Fly.App("Site"));
527 try {
528 const proxy = yield* readinessProxy();
529 const initial = yield* readinessActor(
530 stack,
531 `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
532 file,
533 proxy,
534 );
535 const first = yield* checkedDeployment(initial, proxy, "one", mode, 3, [0]);
536 yield* idleAll(site.appName, first.machineIds, mode);
537 const source = (yield* census(site.appName))[0]!;
538 const metadata = source.config!.metadata!;
539 const attemptBegin = proxy.events.length;
540 yield* Effect.gen(function* () {
541 try {
542 const gate = yield* Deferred.make<string>();
543 const actor = yield* readinessActor(
544 stack,
545 `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
546 file,
547 proxy,
548 (event) =>
549 event.method === "POST" &&
550 event.path ===
551 `/v1/apps/${site.appName}/machines/${event.machineId}` &&
552 event.metadata?.["alchemy.replica"] === "1" &&
553 event.metadata["alchemy.fqn"] === metadata["alchemy.fqn"] &&
554 event.metadata["alchemy.generation"] !==
555 metadata["alchemy.generation"] &&
556 event.metadata["alchemy.readiness-roles"] === "run,run,idle" &&
557 event.metadata["alchemy.idle-policy-restored"] === "true" &&
558 event.phase === "promoting"
559 ? Deferred.succeed(gate, event.machineId!).pipe(
560 Effect.andThen(Effect.never),
561 )
562 : Effect.void,
563 );
564 const attempt = yield* deployWorker(
565 actor,
566 "two",
567 topology(mode, 3, 2),
568 ).pipe(Effect.scoped, Effect.forkScoped);
569 const barrierMachine = yield* Deferred.await(gate).pipe(
570 Effect.timeout("180 seconds"),
571 );
572 const pending = yield* census(site.appName);
573 expect(
574 pending.filter((machine) => first.machineIds.includes(machine.id!)),
575 ).toHaveLength(3);
576 expect(
577 proxy.events.some((event) => retires(event, first.machineIds)),
578 ).toBe(false);
579 const candidates = yield* Effect.forEach(
580 pending.filter((machine) => !first.machineIds.includes(machine.id!)),
581 (machine) =>
582 machines.getMachine({
583 app_name: site.appName,
584 machine_id: machine.id!,
585 }),
586 );
587 expect(candidates).toHaveLength(3);
588 expect(
589 candidates.every(
590 (machine) =>
591 machine.config?.metadata?.["alchemy.phase"] === "promoting",
592 ),
593 ).toBe(true);
594 const pendingSlots = assertSlots(candidates, [0, 1]);
595 expect(
596 pendingSlots[0]!.config?.metadata?.["alchemy.idle-policy-restored"],
597 ).toBe("true");
598 expect(
599 pendingSlots[1]!.config?.metadata?.["alchemy.idle-policy-restored"],
600 ).toBe("false");
601 expect(
602 proxy.readiness.some(
603 (event) =>
604 event.stage === "forwarded" &&
605 event.method === "GET" &&
606 event.machineId === pendingSlots[0]!.id &&
607 event.instanceId === pendingSlots[0]!.instance_id &&
608 event.metadata?.["alchemy.idle-policy-restored"] === "true" &&
609 event.phase === "promoting" &&
610 event.state === "started" &&
611 readinessChecksPassing(event.checks, [
612 "ready",
613 "servicecheck-00-http-80",
614 ]),
615 ),
616 ).toBe(true);
617 expect(barrierMachine).toBe(pendingSlots[1]!.id);
618 expect(
619 autostopMode(pendingSlots[0]!.config?.services?.[0]?.autostop),
620 ).toBe(mode);
621 expect(
622 autostopMode(pendingSlots[1]!.config?.services?.[0]?.autostop),
623 ).toBe("off");
624 expect(
625 proxy.events
626 .slice(attemptBegin)
627 .some(
628 (event) =>
629 event.method === "POST" &&
630 event.path.endsWith(`/machines/${barrierMachine}`) &&
631 event.phase === "promoting",
632 ),
633 ).toBe(false);
634 const read = yield* observeReplicaSet({
635 appName: site.appName,
636 id: "Worker",
637 type: "Fly.Machine",
638 fqn: metadata["alchemy.fqn"]!,
639 resourceInstanceId: metadata["alchemy.instance"]!,
640 baseName: first.baseName,
641 machineIds: first.machineIds,
642 });
643 expect(read?.machineIds).toEqual(first.machineIds);
644 expect(read?.count).toBe(3);
645 expect(read?.rolloutPending).toBe(true);
646 const interruption = yield* Fiber.interrupt(attempt).pipe(
647 Effect.forkScoped,
648 );
649 yield* Effect.yieldNow;
650 yield* Effect.sync(() => {
651 proxy.clear();
652 proxy.release();
653 });
654 yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds"));
655 expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true);
656 expect(
657 proxy.events
658 .slice(attemptBegin)
659 .some(
660 (event) =>
661 first.machineIds.includes(event.machineId!) &&
662 event.path.endsWith("/start"),
663 ),
664 ).toBe(false);
665 expect(
666 proxy.readiness.some(
667 (event) =>
668 event.stage === "request" &&
669 event.path.endsWith("/metadata") &&
670 event.phase === "active" &&
671 candidates.some((machine) => machine.id === event.machineId),
672 ),
673 ).toBe(false);
674 expect(
675 proxy.events.some((event) => retires(event, first.machineIds)),
676 ).toBe(false);
677 const resumed = yield* readinessActor(
678 stack,
679 `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
680 file,
681 proxy,
682 );
683 const begin = proxy.readiness.length;
684 const recovered = yield* deployWorker(
685 resumed,
686 "two",
687 topology(mode, recoveredCount, 2),
688 ).pipe(Effect.scoped);
689 expect(recovered.machineIds).toHaveLength(recoveredCount);
690 if (recoveredCount === 3) {
691 expect([...recovered.machineIds].sort()).toEqual(
692 candidates.map((machine) => machine.id).sort(),
693 );
694 } else {
695 expect(
696 recovered.machineIds.every(
697 (id) => !pending.some((machine) => machine.id === id),
698 ),
699 ).toBe(true);
700 }
701 const runningSlots = recoveredCount === 1 ? [0] : [0, 1];
702 const live = yield* assertTopology(
703 site.appName,
704 recovered.machineIds,
705 mode,
706 runningSlots,
707 2,
708 );
709 assertReadinessCommit(
710 proxy.readiness.slice(begin),
711 pending
712 .map((machine) => machine.id!)
713 .filter((id) => !recovered.machineIds.includes(id)),
714 live,
715 runningSlots,
716 ["ready", "servicecheck-00-http-80"],
717 true,
718 );
719 const unchanged = yield* deployWorker(
720 resumed,
721 "two",
722 topology(mode, recoveredCount, 2),
723 ).pipe(Effect.scoped);
724 expect(unchanged.machineIds).toEqual(recovered.machineIds);
725 const committed = yield* observeReplicaSet({
726 appName: site.appName,
727 id: "Worker",
728 type: "Fly.Machine",
729 fqn: metadata["alchemy.fqn"]!,
730 resourceInstanceId: metadata["alchemy.instance"]!,
731 machineIds: recovered.machineIds,
732 });
733 expect(committed?.rolloutPending).toBe(false);
734 expect(committed?.machineIds).toEqual(recovered.machineIds);
735 } finally {
736 yield* Effect.sync(() => {
737 proxy.clear();
738 proxy.release();
739 });
740 }
741 }).pipe(Effect.scoped);
742 } finally {
743 yield* stack.destroy();
744 yield* assertAppGone(site.appName);
745 }
746 }).pipe(Effect.scoped),
747 { timeout: 900_000 },
748 );
749 }
750
751 test.provider(
752 `S11 public traffic autostarts ${mode} and Fly automatically returns it to idle within 15 minutes`,
753 (stack) =>
754 Effect.gen(function* () {
755 yield* stack.destroy();
756 const site = yield* stack.deploy(Fly.App("Site"));
757 try {
758 const output = yield* stack.deploy(
759 Effect.gen(function* () {
760 const app = yield* Fly.App("Site");
761 yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
762 return yield* Fly.Machine("Worker", {
763 app,
764 image: "nginx:alpine",
765 deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
766 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
767 services: [
768 {
769 protocol: "tcp",
770 internalPort: 80,
771 autostop: mode,
772 autostart: true,
773 minMachinesRunning: 0,
774 ports: [{ port: 80, handlers: ["http"] }],
775 checks: [checks.ready],
776 },
777 ],
778 });
779 }),
780 );
781 yield* idleAll(site.appName, output.machineIds, mode);
782 const client = yield* HttpClient.HttpClient;
783 yield* Effect.gen(function* () {
784 const response = yield* client.get(`http://${site.appName}.fly.dev`);
785 const body = yield* response.text;
786 if (response.status !== 200 || !body.includes("Welcome to nginx")) {
787 return yield* Effect.fail(
788 new Error(`Public Fly route is not ready: ${response.status}`),
789 );
790 }
791 expect(response.status).toBe(200);
792 expect(body).toContain("Welcome to nginx");
793 }).pipe(
794 Effect.retry({ schedule: Schedule.spaced("3 seconds"), times: 20 }),
795 Effect.timeout("90 seconds"),
796 );
797 yield* waitState(site.appName, output.machineId, "started");
798 const started = yield* Clock.currentTimeMillis;
799 const idleState = mode === "stop" ? "stopped" : "suspended";
800 // Only native GETs follow the last public request; no stop, suspend, or update substitutes.
801 const idle = yield* machines
802 .getMachine({ app_name: site.appName, machine_id: output.machineId })
803 .pipe(
804 Effect.tap((machine) =>
805 Effect.logInfo("Automatic Fly idle observation", {
806 mode,
807 state: machine.state,
808 instanceId: machine.instance_id,
809 }),
810 ),
811 Effect.repeat({
812 schedule: Schedule.spaced("10 seconds"),
813 times: 90,
814 until: (machine) => machine.state === idleState,
815 }),
816 Effect.timeout("15 minutes"),
817 );
818 expect(idle.state).toBe(idleState);
819 expect(idle.cordoned).toBe(false);
820 expect(autostopMode(idle.config?.services?.[0]?.autostop)).toBe(mode);
821 expect(idle.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
822 yield* Effect.logInfo("Automatic return to idle observed", {
823 mode,
824 elapsedMs: (yield* Clock.currentTimeMillis) - started,
825 });
826 expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual(
827 output.machineIds,
828 );
829 } finally {
830 yield* stack.destroy();
831 yield* assertAppGone(site.appName);
832 }
833 }),
834 { tags: ["provider:fly:ipassignment"], timeout: 1_200_000 },
835 );
836 }
837 });
838 },
839 );
840
841 describe.sequential(
842 "idle capacity",
843 {
844 tags: [
845 "provider:fly",
846 "provider:fly:app",
847 "provider:fly:machine",
848 "provider:fly:service",
849 "live",
850 ],
851 },
852 () => {
853 let endpoint: string | undefined;
854 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
855
856 describe.sequential("idle topology", () => {
857 for (const mode of ["stop", "suspend"] as const) {
858 for (const allIdle of [false, true]) {
859 test.provider(
860 `S05 S11 ${mode} ${allIdle ? "all-idle" : "mixed"} replacement keeps an idle nonrepresentative`,
861 (stack) =>
862 Effect.gen(function* () {
863 yield* stack.destroy();
864 const props = {
865 count: 2,
866 services: [
867 {
868 protocol: "tcp",
869 internalPort: 80,
870 autostop: mode,
871 autostart: true,
872 minMachinesRunning: 0,
873 ports: [{ port: 80, handlers: ["http"] }],
874 checks: [checks.ready],
875 },
876 ],
877 };
878 const initial = yield* deployWorker(stack, "one", {
879 ...props,
880 deploy: { strategy: "rolling" },
881 });
882 for (const id of allIdle ? initial.machineIds : initial.machineIds.slice(1)) {
883 const target = { app_name: initial.appName, machine_id: id };
884 yield* mode === "stop"
885 ? machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" })
886 : machines.suspendMachine(target);
887 const observed = yield* machines.getMachine(target).pipe(
888 Effect.repeat({
889 schedule: Schedule.spaced("2 seconds"),
890 times: 8,
891 until: (machine) =>
892 machine.state === (mode === "stop" ? "stopped" : "suspended"),
893 }),
894 );
895 expect(observed.state).toBe(mode === "stop" ? "stopped" : "suspended");
896 }
897 const proxy = yield* transportProxy();
898 yield* Effect.sync(() => {
899 endpoint = proxy.url;
900 });
901 const next = yield* deployWorker(stack, "two", props);
902 expect(next.machineIds).toHaveLength(2);
903 const live = yield* census(initial.appName);
904 expect(live.map((machine) => machine.id).sort()).toEqual(
905 [...next.machineIds].sort(),
906 );
907 expect(
908 live.every((machine) => {
909 const autostop = machine.config?.services?.[0]?.autostop;
910 return (autostop === true ? "stop" : autostop) === mode;
911 }),
912 ).toBe(true);
913 expect(
914 live.filter((machine) => machine.state === "started").length,
915 ).toBeLessThanOrEqual(1);
916 expect(
917 live.some((machine) => ["stopped", "suspended"].includes(machine.state!)),
918 ).toBe(true);
919 expect(
920 proxy.events.some(
921 (event) =>
922 initial.machineIds.includes(event.machineId!) &&
923 event.path.endsWith("/start"),
924 ),
925 ).toBe(false);
926 const same = yield* deployWorker(stack, "two", props);
927 expect(same.machineIds).toEqual(next.machineIds);
928 yield* Effect.sync(() => {
929 endpoint = undefined;
930 });
931 yield* stack.destroy();
932 yield* assertAppGone(initial.appName);
933 }).pipe(
934 Effect.scoped,
935 Effect.ensuring(
936 Effect.sync(() => {
937 endpoint = undefined;
938 }),
939 ),
940 ),
941 { timeout: 300_000 },
942 );
943 }
944 }
945 });
946 },
947 );
948
949 describe.sequential(
950 "post-promotion health",
951 {
952 tags: [
953 "provider:fly",
954 "provider:fly:app",
955 "provider:fly:ipassignment",
956 "provider:fly:machine",
957 "provider:fly:secret",
958 "provider:fly:service",
959 "live",
960 ],
961 },
962 () => {
963 let endpoint: string | undefined;
964 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
965
966 describe.sequential("post-promotion health", () => {
967 for (const changed of [false, true]) {
968 test.provider(
969 `F09 a real post-uncordon health flip preserves both generations then recovers ${changed ? "changed" : "same"} desired state`,
970 (stack) =>
971 Effect.gen(function* () {
972 yield* stack.destroy();
973 const readiness = yield* makeReadinessControl();
974 const initial = yield* readiness.deployWorker(stack, "one");
975 const proxy = yield* transportProxy();
976 yield* Effect.sync(() => {
977 endpoint = proxy.url;
978 proxy.arm({
979 match: (event) => event.path.endsWith("/uncordon"),
980 action: "hold-response",
981 remaining: 1,
982 });
983 });
984 const update = yield* readiness
985 .deployWorker(stack, "two")
986 .pipe(Effect.result, Effect.forkScoped);
987 const promoted = yield* proxy.wait(
988 (event) =>
989 event.stage === "held" &&
990 event.path.endsWith("/uncordon") &&
991 event.status! < 300,
992 );
993 expect(promoted.machineId).toBeDefined();
994 yield* readiness.turnOff(initial.appName, promoted.machineId!);
995 yield* Effect.sync(proxy.release);
996 const result = yield* Fiber.join(update).pipe(Effect.timeout("60 seconds"));
997 expect(Result.isFailure(result)).toBe(true);
998 if (Result.isFailure(result))
999 expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
1000 const live = yield* census(initial.appName);
1001 expect(live.map((machine) => machine.id).sort()).toEqual(
1002 [initial.machineId, promoted.machineId!].sort(),
1003 );
1004 expect(live.every((machine) => machine.cordoned === false)).toBe(true);
1005 const pending = live.find((machine) => machine.id === promoted.machineId)!;
1006 const metadata = pending.config!.metadata!;
1007 expect(metadata["alchemy.phase"]).toBe("validating");
1008 expect(metadata["alchemy.checked-instance"]).toBeUndefined();
1009 const read = yield* observeReplicaSet({
1010 appName: initial.appName,
1011 id: "Worker",
1012 type: "Fly.Machine",
1013 fqn: metadata["alchemy.fqn"]!,
1014 resourceInstanceId: metadata["alchemy.instance"]!,
1015 machineIds: initial.machineIds,
1016 });
1017 expect(read?.machineIds).toEqual(initial.machineIds);
1018 expect(read?.rolloutPending).toBe(true);
1019 expect(
1020 proxy.events.some(
1021 (event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path),
1022 ),
1023 ).toBe(false);
1024 if (!changed) {
1025 yield* repairReadiness(initial.appName, promoted.machineId!);
1026 }
1027 const recovered = yield* readiness.deployWorker(stack, changed ? "three" : "two");
1028 if (changed) expect(recovered.machineId).not.toBe(promoted.machineId);
1029 else expect(recovered.machineId).toBe(promoted.machineId);
1030 yield* assertCommitted(initial.appName, recovered.machineIds);
1031 yield* Effect.sync(() => {
1032 endpoint = undefined;
1033 });
1034 yield* stack.destroy();
1035 yield* assertAppGone(initial.appName);
1036 }).pipe(
1037 Effect.scoped,
1038 Effect.ensuring(
1039 Effect.sync(() => {
1040 endpoint = undefined;
1041 }),
1042 ),
1043 ),
1044 { timeout: 300_000 },
1045 );
1046 }
1047 });
1048 },
1049 );
1050
1051 describe.sequential(
1052 "ownership",
1053 {
1054 tags: [
1055 "provider:fly",
1056 "provider:fly:app",
1057 "provider:fly:machine",
1058 "provider:fly:service",
1059 "live",
1060 ],
1061 },
1062 () => {
1063 const { test } = Test.make({ providers: Fly.providers() });
1064
1065 test.provider(
1066 "S09 foreign routed Machine and sibling survive rollout and stale-instance deletion",
1067 (stack) =>
1068 Effect.gen(function* () {
1069 yield* stack.destroy();
1070 const deploy = (version: string) =>
1071 stack.deploy(
1072 Effect.gen(function* () {
1073 const app = yield* Fly.App("Site");
1074 const sibling = yield* Fly.Machine("Sibling", {
1075 app,
1076 image: "nginx:alpine",
1077 env: { VERSION: "sibling" },
1078 });
1079 const worker = yield* Fly.Machine("Worker", {
1080 app,
1081 image: "nginx:alpine",
1082 env: { VERSION: version },
1083 checks,
1084 deploy: { strategy: "bluegreen", healthTimeout: "20 seconds" },
1085 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
1086 });
1087 return { sibling, worker };
1088 }),
1089 );
1090 const initial = yield* deploy("one");
1091 const foreign = yield* machines.createMachine({
1092 app_name: initial.worker.appName,
1093 name: "foreign-routed",
1094 region: "iad",
1095 config: {
1096 image: "nginx:alpine",
1097 env: { VERSION: "foreign" },
1098 services: [
1099 {
1100 protocol: "tcp",
1101 internal_port: 80,
1102 ports: [{ port: 80, handlers: ["http"] }],
1103 },
1104 ],
1105 },
1106 });
1107 const removeForeign = machines
1108 .deleteMachine({
1109 app_name: initial.worker.appName,
1110 machine_id: foreign.id!,
1111 force: true,
1112 })
1113 .pipe(Effect.catchTag("NotFound", () => Effect.void));
1114 yield* Effect.gen(function* () {
1115 const next = yield* deploy("two");
1116 expect(next.worker.machineId).not.toBe(initial.worker.machineId);
1117 expect(next.sibling.machineId).toBe(initial.sibling.machineId);
1118 const before = yield* census(initial.worker.appName);
1119 expect(before.map((machine) => machine.id).sort()).toEqual(
1120 [next.worker.machineId, next.sibling.machineId, foreign.id!].sort(),
1121 );
1122 expect(before.find((machine) => machine.id === foreign.id)?.cordoned).toBe(false);
1123 const owned = before.find((machine) => machine.id === next.worker.machineId)!;
1124 yield* deleteReplicaSet({
1125 appName: next.worker.appName,
1126 id: "Worker",
1127 type: "Fly.Machine",
1128 fqn: owned.config!.metadata!["alchemy.fqn"]!,
1129 resourceInstanceId: "stale-instance-must-not-delete-successor",
1130 machineIds: next.worker.machineIds,
1131 volumeIds: [],
1132 });
1133 expect(
1134 (yield* census(initial.worker.appName)).map((machine) => machine.id).sort(),
1135 ).toEqual(before.map((machine) => machine.id).sort());
1136 }).pipe(Effect.ensuring(removeForeign.pipe(Effect.orDie)));
1137 yield* stack.destroy();
1138 yield* assertAppGone(initial.worker.appName);
1139 }),
1140 { timeout: 300_000 },
1141 );
1142
1143 test.provider(
1144 "S09 F11 stale output and a same-logical-ID foreign FQN cannot authorize successor deletion",
1145 (stack) =>
1146 Effect.gen(function* () {
1147 yield* stack.destroy();
1148 const deploy = (name: string, version: string) =>
1149 stack.deploy(
1150 Effect.gen(function* () {
1151 const app = yield* Fly.App("Site");
1152 return yield* Fly.Machine("Worker", {
1153 app,
1154 name,
1155 image: "nginx:alpine",
1156 checks,
1157 env: { VERSION: version },
1158 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
1159 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
1160 });
1161 }),
1162 );
1163 const initial = yield* deploy("stale-original", "one");
1164 const old = yield* machines.getMachine({
1165 app_name: initial.appName,
1166 machine_id: initial.machineId,
1167 });
1168 const oldMetadata = old.config!.metadata!;
1169 const collision = yield* machines.createMachine({
1170 app_name: initial.appName,
1171 name: "fqn-collision",
1172 region: "iad",
1173 config: {
1174 ...old.config,
1175 metadata: {
1176 ...oldMetadata,
1177 "alchemy.fqn": `${oldMetadata["alchemy.fqn"]}/OtherScope`,
1178 },
1179 },
1180 });
1181 yield* Effect.gen(function* () {
1182 const next = yield* deploy("stale-original", "two");
1183 const refreshed = yield* observeReplicaSet({
1184 appName: initial.appName,
1185 id: "Worker",
1186 type: "Fly.Machine",
1187 fqn: oldMetadata["alchemy.fqn"]!,
1188 resourceInstanceId: oldMetadata["alchemy.instance"]!,
1189 baseName: initial.baseName,
1190 machineIds: [...initial.machineIds, collision.id!],
1191 });
1192 expect(refreshed?.machineIds).toEqual(next.machineIds);
1193 expect(refreshed?.rolloutPending).toBe(false);
1194 const successor = yield* deploy("stale-successor", "three");
1195 const current = yield* machines.getMachine({
1196 app_name: initial.appName,
1197 machine_id: successor.machineId,
1198 });
1199 expect(current.config?.metadata?.["alchemy.instance"]).not.toBe(
1200 oldMetadata["alchemy.instance"],
1201 );
1202 // Even an ID cache containing the new Machine grants no old-lineage authority.
1203 yield* deleteReplicaSet({
1204 appName: initial.appName,
1205 id: "Worker",
1206 type: "Fly.Machine",
1207 fqn: oldMetadata["alchemy.fqn"]!,
1208 resourceInstanceId: oldMetadata["alchemy.instance"]!,
1209 machineIds: [
1210 ...initial.machineIds,
1211 ...next.machineIds,
1212 ...successor.machineIds,
1213 collision.id!,
1214 ],
1215 volumeIds: [],
1216 });
1217 const live = yield* census(initial.appName);
1218 expect(live.map((machine) => machine.id).sort()).toEqual(
1219 [successor.machineId, collision.id!].sort(),
1220 );
1221 const foreign = live.find((machine) => machine.id === collision.id)!;
1222 expect(foreign.config?.metadata?.["alchemy.fqn"]).toBe(
1223 `${oldMetadata["alchemy.fqn"]}/OtherScope`,
1224 );
1225 expect(foreign.cordoned).toBe(false);
1226 }).pipe(
1227 Effect.ensuring(
1228 machines
1229 .deleteMachine({
1230 app_name: initial.appName,
1231 machine_id: collision.id!,
1232 force: true,
1233 })
1234 .pipe(
1235 Effect.catchTag("NotFound", () => Effect.void),
1236 Effect.orDie,
1237 ),
1238 ),
1239 );
1240 yield* stack.destroy();
1241 yield* assertAppGone(initial.appName);
1242 }),
1243 { timeout: 600_000 },
1244 );
1245 },
1246 );
1247
1248 describe.sequential(
1249 "deployment",
1250 {
1251 tags: [
1252 "provider:fly",
1253 "provider:fly:app",
1254 "provider:fly:machine",
1255 "provider:fly:service",
1256 "live",
1257 ],
1258 },
1259 () => {
1260 const { test } = Test.make({ providers: Fly.providers() });
1261
1262 test.provider(
1263 "S01 bluegreen worker checks, promotion, replacement, and graceful teardown",
1264 (stack) =>
1265 Effect.gen(function* () {
1266 yield* stack.destroy();
1267 const deploy = (version: string, path = "/") =>
1268 stack.deploy(
1269 Effect.gen(function* () {
1270 const app = yield* Fly.App("Site");
1271 return yield* Fly.Machine("Worker", {
1272 app,
1273 name: "bluegreen-worker-long-name-base",
1274 image: "nginx:alpine",
1275 env: { VERSION: version },
1276 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
1277 shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
1278 checks: {
1279 ready: { type: "http", port: 80, path, interval: "2s", timeout: "1s" },
1280 },
1281 });
1282 }),
1283 );
1284 const first = yield* deploy("one");
1285 const committed = yield* machines.getMachine({
1286 app_name: first.appName,
1287 machine_id: first.machineId,
1288 });
1289 // Active commit metadata can reset reports after the provider's readiness validation.
1290 const machine = yield* waitHealthy(first.appName, committed, 30_000);
1291 expect(machine.instance_id).toBe(committed.instance_id);
1292 yield* Effect.logInfo("Observed promoted worker", {
1293 checks: machine.checks?.map(({ name, status }) => ({ name, status })),
1294 configured: machine.config?.checks,
1295 metadata: machine.config?.metadata,
1296 });
1297 expect(machine.cordoned).toBe(false);
1298 expect(
1299 machine.checks?.some(
1300 (check) => check.name === "ready" && check.status === "passing",
1301 ),
1302 ).toBe(true);
1303 expect(machine.config?.metadata?.["alchemy.phase"]).toBe("active");
1304 expect(first.name.length).toBeLessThanOrEqual(30);
1305 const second = yield* deploy("two");
1306 expect(second.machineId).not.toBe(first.machineId);
1307 expect(second.baseName).toBe(first.baseName);
1308 const listed = yield* machines.listMachines({ app_name: first.appName });
1309 expect(
1310 listed
1311 .filter((machine) => machine.state !== "destroyed")
1312 .map((machine) => machine.id),
1313 ).toEqual([second.machineId]);
1314 yield* stack.destroy();
1315 yield* assertAppGone(first.appName);
1316 }),
1317 { timeout: 180_000 },
1318 );
1319
1320 test.provider(
1321 "S04 unhealthy replacement preserves the old routed ID and cleans unpromoted candidates",
1322 (stack) =>
1323 Effect.gen(function* () {
1324 yield* stack.destroy();
1325 const initial = yield* deployWorker(stack, "one");
1326 const failed = yield* deployWorker(stack, "broken", {
1327 checks: { ready: { ...checks.ready, path: "/missing" } },
1328 deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
1329 }).pipe(Effect.result);
1330 expect(Result.isFailure(failed)).toBe(true);
1331 if (Result.isFailure(failed))
1332 expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
1333 const live = yield* census(initial.appName);
1334 expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
1335 expect(live[0]!.state).toBe("started");
1336 expect(live[0]!.cordoned).toBe(false);
1337 yield* stack.destroy();
1338 yield* assertAppGone(initial.appName);
1339 }),
1340 { timeout: 180_000 },
1341 );
1342 },
1343 );
1344
1345 describe.sequential(
1346 "promotion faults",
1347 {
1348 tags: [
1349 "provider:fly",
1350 "provider:fly:app",
1351 "provider:fly:machine",
1352 "provider:fly:service",
1353 "live",
1354 ],
1355 },
1356 () => {
1357 let endpoint: string | undefined;
1358 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
1359
1360 test.provider(
1361 "F03 F08 partial promotion loses the first uncordon response without rolling back serving green",
1362 (stack) =>
1363 Effect.gen(function* () {
1364 yield* stack.destroy();
1365 const initial = yield* deployWorker(stack, "one", { count: 2 });
1366 const proxy = yield* transportProxy();
1367 const match = (event: { path: string }) => event.path.endsWith("/uncordon");
1368 yield* Effect.sync(() => {
1369 endpoint = proxy.url;
1370 proxy.arm({ match, action: "drop-response", remaining: 1 });
1371 proxy.arm({ match, action: "cut-request", remaining: Infinity });
1372 });
1373 try {
1374 const failed = yield* deployWorker(stack, "two", { count: 2 }).pipe(
1375 Effect.timeout("120 seconds"),
1376 Effect.result,
1377 );
1378 expect(Result.isFailure(failed)).toBe(true);
1379 const lost = proxy.events.find(
1380 (event) => event.stage === "dropped" && event.path.endsWith("/uncordon"),
1381 );
1382 expect(lost?.status).toBeGreaterThanOrEqual(200);
1383 expect(lost?.status).toBeLessThan(300);
1384 const live = yield* census(initial.appName);
1385 expect(
1386 initial.machineIds.every((id) =>
1387 live.some((machine) => machine.id === id && machine.cordoned === false),
1388 ),
1389 ).toBe(true);
1390 expect(live.find((machine) => machine.id === lost!.machineId)?.cordoned).toBe(
1391 false,
1392 );
1393 const candidateIds = live
1394 .filter((machine) => !initial.machineIds.includes(machine.id!))
1395 .map((machine) => machine.id!);
1396 expect(candidateIds).toHaveLength(2);
1397 expect(
1398 proxy.events.some(
1399 (event) =>
1400 event.method === "DELETE" &&
1401 /\/machines\/[^/]+$/.test(event.path) &&
1402 candidateIds.includes(event.machineId!),
1403 ),
1404 ).toBe(false);
1405 yield* Effect.sync(proxy.clear);
1406 const recovered = yield* deployWorker(stack, "two", { count: 2 });
1407 expect([...recovered.machineIds].sort()).toEqual(candidateIds.sort());
1408 yield* assertCommitted(initial.appName, recovered.machineIds);
1409 } finally {
1410 yield* Effect.sync(() => {
1411 endpoint = undefined;
1412 proxy.clear();
1413 });
1414 }
1415 yield* stack.destroy();
1416 yield* assertAppGone(initial.appName);
1417 }).pipe(
1418 Effect.scoped,
1419 Effect.ensuring(
1420 Effect.sync(() => {
1421 endpoint = undefined;
1422 }),
1423 ),
1424 ),
1425 { timeout: 300_000 },
1426 );
1427 },
1428 );
1429
1430 describe.sequential(
1431 "transport",
1432 {
1433 tags: [
1434 "provider:fly",
1435 "provider:fly:app",
1436 "provider:fly:machine",
1437 "provider:fly:service",
1438 "live",
1439 ],
1440 },
1441 () => {
1442 let endpoint: string | undefined;
1443 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
1444
1445 test.provider(
1446 "F02 S07 harness forwards real Fly responses, loses completed responses and cuts connections",
1447 (stack) =>
1448 Effect.gen(function* () {
1449 yield* stack.destroy();
1450 const app = yield* stack.deploy(Fly.App("Site"));
1451 const proxy = yield* transportProxy();
1452 yield* Effect.sync(() => {
1453 endpoint = proxy.url;
1454 });
1455 try {
1456 const request = machines.getApp({ app_name: app.appName }).pipe(Retry.none);
1457 const forwarded = yield* request;
1458 expect(forwarded.name).toBe(app.appName);
1459 expect(
1460 proxy.events.some((event) => event.stage === "forwarded" && event.status === 200),
1461 ).toBe(true);
1462 yield* Effect.sync(() =>
1463 proxy.arm({
1464 match: (event) => event.path.endsWith(`/apps/${app.appName}`),
1465 action: "drop-response",
1466 remaining: 1,
1467 }),
1468 );
1469 const dropped = yield* request.pipe(Effect.result);
1470 // The HTTP runtime may transparently replay an idempotent GET after a socket reset.
1471 if (Result.isFailure(dropped)) expect(dropped.failure._tag).toBe("HttpClientError");
1472 else expect(dropped.success.name).toBe(app.appName);
1473 expect(
1474 proxy.events.filter((event) => event.stage === "dropped" && event.status === 200),
1475 ).toHaveLength(1);
1476 yield* Effect.sync(() =>
1477 proxy.arm({ match: () => true, action: "cut-request", remaining: Infinity }),
1478 );
1479 const cut = yield* request.pipe(Effect.result);
1480 expect(Result.isFailure(cut)).toBe(true);
1481 const cutEvent = proxy.events.find((event) => event.stage === "cut")!;
1482 expect(
1483 proxy.events.some(
1484 (event) => event.sequence === cutEvent.sequence && event.stage === "completed",
1485 ),
1486 ).toBe(false);
1487 yield* Effect.sync(() => {
1488 proxy.clear();
1489 proxy.arm({ match: () => true, action: "hold-response", remaining: 1 });
1490 });
1491 const child = yield* request.pipe(Effect.forkScoped);
1492 const held = yield* proxy.wait((event) => event.stage === "held");
1493 expect(held.status).toBe(200);
1494 yield* Effect.sync(proxy.release);
1495 expect((yield* Fiber.join(child).pipe(Effect.timeout("10 seconds"))).name).toBe(
1496 app.appName,
1497 );
1498 expect((yield* request).name).toBe(app.appName);
1499 } finally {
1500 yield* Effect.sync(() => {
1501 endpoint = undefined;
1502 proxy.clear();
1503 proxy.release();
1504 });
1505 }
1506 yield* stack.destroy();
1507 yield* assertAppGone(app.appName);
1508 }).pipe(
1509 Effect.scoped,
1510 Effect.ensuring(
1511 Effect.sync(() => {
1512 endpoint = undefined;
1513 }),
1514 ),
1515 ),
1516 { timeout: 180_000 },
1517 );
1518 },
1519 );
1520 });
1521 describe.sequential("shard 2", () => {
1522 describe.sequential(
1523 "workers",
1524 {
1525 tags: [
1526 "provider:fly",
1527 "provider:fly:app",
1528 "provider:fly:ipassignment",
1529 "provider:fly:machine",
1530 "provider:fly:redis",
1531 "provider:fly:secret",
1532 "provider:fly:service",
1533 "live",
1534 ],
1535 },
1536 () => {
1537 const { test } = Test.make({ providers: Fly.providers() });
1538
1539 // Cold gateway/worker publication, readiness, replacement and census can exceed three minutes.
1540 const timeout = 480_000;
1541
1542 describe.sequential("Fly durable worker acceptance", () => {
1543 for (const mode of ["stop-delay", "stop-fail", "stop-hang"]) {
1544 test.provider(
1545 `R05 live ${mode} preserves another worker and bounds managed shutdown`,
1546 (stack) =>
1547 Effect.gen(function* () {
1548 yield* stack.destroy();
1549 const scenario = yield* makeScenario(stack);
1550 yield* Effect.gen(function* () {
1551 const first = yield* scenario.deploy({
1552 version: "one",
1553 timeout: "10 seconds",
1554 signal: "SIGTERM",
1555 runOnly: true,
1556 mode,
1557 workers: 2,
1558 });
1559 const worker = yield* requireValue(first.worker, "worker output");
1560 const machineId = worker.machineId;
1561 const observe = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>
1562 Effect.gen(function* () {
1563 const started = yield* Clock.currentTimeMillis;
1564 yield* Effect.logInfo("R05 phase started", { mode, machineId, phase });
1565 return yield* effect.pipe(
1566 Effect.onExit((exit) =>
1567 Clock.currentTimeMillis.pipe(
1568 Effect.flatMap((finished) =>
1569 Effect.logInfo("R05 phase finished", {
1570 mode,
1571 machineId,
1572 phase,
1573 success: Exit.isSuccess(exit),
1574 elapsedMs: finished - started,
1575 }),
1576 ),
1577 ),
1578 ),
1579 );
1580 });
1581 yield* observe(
1582 "both-workers-ready-ledger",
1583 scenario.wait(
1584 first.ledgerUrl,
1585 (ledger) =>
1586 ledger.events.filter(
1587 (event) =>
1588 event.machine === machineId && event.event === "worker-ready",
1589 ).length === 2,
1590 ),
1591 );
1592 yield* observe(
1593 "stop-machine-request",
1594 machines.stopMachine({
1595 app_name: first.workerApp.appName,
1596 machine_id: machineId,
1597 }),
1598 );
1599 const stopped = yield* observe(
1600 "stopped-machine-observation",
1601 machines
1602 .getMachine({ app_name: first.workerApp.appName, machine_id: machineId })
1603 .pipe(
1604 Effect.repeat({
1605 until: (value) => value.state === "stopped",
1606 times: 8,
1607 schedule: Schedule.spaced("2 seconds"),
1608 }),
1609 ),
1610 );
1611 expect(stopped.state).toBe("stopped");
1612 const ledger = yield* observe(
1613 "post-stop-ledger-snapshot",
1614 scenario.snapshot(first.ledgerUrl),
1615 );
1616 assertOrder(ledger.events, machineId, "stopped", "drained", "b");
1617 assertOrder(ledger.events, machineId, "drained", "client-released", "b");
1618 const stopStarted = yield* requireValue(
1619 ledger.events.find(
1620 (event) => event.machine === machineId && event.event === "stop-started",
1621 ),
1622 "worker stop-started event",
1623 );
1624 const independentRelease = yield* requireValue(
1625 ledger.events.find(
1626 (event) =>
1627 event.machine === machineId &&
1628 event.worker === "b" &&
1629 event.event === "client-released",
1630 ),
1631 "worker b client release event",
1632 );
1633 expect(independentRelease.at - stopStarted.at).toBeGreaterThanOrEqual(0);
1634 expect(independentRelease.at - stopStarted.at).toBeLessThan(10_000);
1635 const selected = ledger.events.filter(
1636 (event) => event.machine === machineId && event.worker === "a",
1637 );
1638 if (mode === "stop-delay") {
1639 const a = yield* requireValue(
1640 selected.find((event) => event.event === "stopped"),
1641 "worker a stopped event",
1642 );
1643 const b = yield* requireValue(
1644 ledger.events.find(
1645 (event) =>
1646 event.machine === machineId &&
1647 event.worker === "b" &&
1648 event.event === "stopped",
1649 ),
1650 "worker b stopped event",
1651 );
1652 expect(a.at - b.at).toBeGreaterThan(700);
1653 assertOrder(ledger.events, machineId, "stopped", "drained", "a");
1654 assertStopped(ledger.events, machineId);
1655 } else {
1656 expect(
1657 selected.some(
1658 (event) => event.event === "stopped" || event.event === "drained",
1659 ),
1660 ).toBe(false);
1661 expect(
1662 selected.some(
1663 (event) =>
1664 event.event === (mode === "stop-fail" ? "stop-failed" : "stop-started"),
1665 ),
1666 ).toBe(true);
1667 }
1668 if (mode === "stop-hang") {
1669 expect(
1670 selected.some(
1671 (event) =>
1672 event.event === "work-closed" || event.event === "client-released",
1673 ),
1674 ).toBe(false);
1675 }
1676 expect(
1677 ledger.events.some(
1678 (event) => event.machine === machineId && event.event === "shared-closed",
1679 ),
1680 ).toBe(mode !== "stop-hang");
1681 const exit = stopped.events?.find((event) => event.type === "exit")?.request as
1682 | { exit_event?: { exit_code?: number } }
1683 | undefined;
1684 yield* Effect.logInfo("Observed managed worker exit", {
1685 mode,
1686 exitCode: exit?.exit_event?.exit_code,
1687 });
1688 expect(exit?.exit_event?.exit_code).toBe(mode === "stop-delay" ? 0 : 1);
1689 }).pipe(Effect.ensuring(scenario.cleanup));
1690 }),
1691 { timeout },
1692 );
1693 }
1694
1695 test.provider(
1696 "R07 authenticated durable Redis Streams gateway",
1697 (stack) =>
1698 Effect.gen(function* () {
1699 yield* stack.destroy();
1700 const scenario = yield* makeScenario(stack);
1701 yield* Effect.gen(function* () {
1702 const first = yield* scenario.deploy({
1703 version: "one",
1704 timeout: "30 seconds",
1705 signal: "SIGTERM",
1706 runOnly: true,
1707 gatewayOnly: true,
1708 });
1709 expect((yield* HttpClient.post(first.ledgerUrl)).status).toBe(401);
1710 const added = yield* scenario
1711 .call(first.ledgerUrl, "enqueue", ["durable-probe", "quick"])
1712 .pipe(Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") }));
1713 expect(typeof added).toBe("string");
1714 expect(
1715 yield* scenario.call(first.ledgerUrl, "enqueue", ["durable-probe", "quick"]),
1716 ).toBe(0);
1717 const claimed = JSON.parse(
1718 String(yield* scenario.call(first.ledgerUrl, "claim", ["probe-client", "one"])),
1719 ) as { id: string; job: string };
1720 expect(claimed.job).toBe("durable-probe");
1721 expect(
1722 yield* scenario.call(first.ledgerUrl, "finish", [
1723 claimed.id,
1724 claimed.job,
1725 "probe-client",
1726 ]),
1727 ).toBe(1);
1728 const ledger = yield* scenario.settle(first.ledgerUrl);
1729 expect(ledger.pending).toBe(0);
1730 expect(ledger.results).toContain("durable-probe");
1731 }).pipe(Effect.ensuring(scenario.cleanup));
1732 }),
1733 { timeout },
1734 );
1735
1736 for (const raw of [false, true]) {
1737 test.provider(
1738 `R07 ${raw ? "raw" : "managed"} private worker checkpoints and reclaims through replacement with producer overlap`,
1739 (stack) =>
1740 Effect.gen(function* () {
1741 yield* stack.destroy();
1742 const scenario = yield* makeScenario(stack);
1743 yield* Effect.gen(function* () {
1744 const options = {
1745 raw,
1746 runOnly: true,
1747 timeout: "30 seconds" as const,
1748 signal: "SIGTERM" as const,
1749 mode: "checkpoint",
1750 };
1751 const first = yield* scenario.deploy({ ...options, version: "one" });
1752 const oldWorker = yield* requireValue(first.worker, "initial worker output");
1753 const oldId = oldWorker.machineId;
1754 yield* scenario.call(first.ledgerUrl, "enqueue", [
1755 "checkpoint-job",
1756 "checkpoint",
1757 ]);
1758 yield* scenario.wait(first.ledgerUrl, (ledger) =>
1759 ledger.events.some(
1760 (event) =>
1761 event.machine === oldId &&
1762 event.event === "claimed" &&
1763 event.job === "checkpoint-job",
1764 ),
1765 );
1766 const second = yield* scenario.deploy({
1767 ...options,
1768 version: "two",
1769 mode: "drain",
1770 signal: "SIGINT",
1771 });
1772 const newWorker = yield* requireValue(
1773 second.worker,
1774 "replacement worker output",
1775 );
1776 const newId = newWorker.machineId;
1777 yield* assertReplacement(first.workerApp.appName, oldId, newId);
1778 const ledger = yield* scenario.settle(first.ledgerUrl);
1779 expect(ledger.results).toContain("checkpoint-job");
1780 expect(ledger.checkpoints).toContain("saved-step-1");
1781 expect(
1782 ledger.events.some(
1783 (event) =>
1784 event.event === "reclaimed" &&
1785 event.machine === newId &&
1786 event.job === "checkpoint-job",
1787 ),
1788 ).toBe(true);
1789 const ack = ledger.events.filter(
1790 (event) => event.event === "acked" && event.job === "checkpoint-job",
1791 );
1792 expect(ack).toHaveLength(1);
1793 expect(ack[0]?.ack).toBe(1);
1794 expect(ack[0]?.fresh).toBe(1);
1795 assertStopped(ledger.events, oldId);
1796 assertOrder(ledger.events, oldId, "stopped", "checkpoint");
1797 assertOrder(ledger.events, oldId, "checkpoint", "client-released");
1798 assertOrder(ledger.events, oldId, "client-released", "shared-closed");
1799 const oldSlots = new Set(
1800 ledger.events
1801 .filter((event) => event.machine === oldId && event.event === "producer")
1802 .map((event) => event.job),
1803 );
1804 const overlaps = ledger.events.filter(
1805 (event) =>
1806 event.machine === newId &&
1807 event.event === "producer" &&
1808 oldSlots.has(event.job),
1809 );
1810 expect(overlaps.length).toBeGreaterThan(0);
1811 for (const event of overlaps) {
1812 expect(
1813 ledger.events.filter(
1814 (row) =>
1815 row.event === "producer" && row.job === event.job && row.fresh === 1,
1816 ),
1817 ).toHaveLength(1);
1818 }
1819 }).pipe(Effect.ensuring(scenario.cleanup));
1820 }),
1821 { timeout },
1822 );
1823 }
1824 });
1825 },
1826 );
1827
1828 describe.sequential(
1829 "concurrency",
1830 {
1831 tags: [
1832 "provider:fly",
1833 "provider:fly:app",
1834 "provider:fly:machine",
1835 "provider:fly:service",
1836 "live",
1837 ],
1838 },
1839 () => {
1840 const file = "test/Fly/BlueGreen.test.ts";
1841 const { test } = Test.make({ providers: Fly.providers() });
1842
1843 test.provider(
1844 "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1845 (stack) =>
1846 Effect.gen(function* () {
1847 yield* stack.destroy();
1848 const initial = yield* deployWorker(stack, "one");
1849 const proxy = yield* transportProxy();
1850 yield* Effect.sync(() =>
1851 proxy.arm({
1852 match: (event) => event.method === "GET" && event.path.endsWith("/machines"),
1853 action: "hold-response",
1854 remaining: 1,
1855 }),
1856 );
1857 yield* Effect.gen(function* () {
1858 const staleActor = yield* engineActor(
1859 stack,
1860 "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1861 file,
1862 proxy.url,
1863 );
1864 const freshActor = yield* engineActor(
1865 stack,
1866 "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1867 file,
1868 );
1869 const delayed = yield* deployWorker(staleActor, "two").pipe(
1870 Effect.scoped,
1871 Effect.result,
1872 Effect.forkScoped,
1873 );
1874 yield* proxy.wait((event) => event.stage === "held" && event.status === 200);
1875 const newer = yield* deployWorker(freshActor, "three");
1876 expect(newer.machineId).not.toBe(initial.machineId);
1877 expect(
1878 proxy.events.some((event) => event.stage === "request" && event.method !== "GET"),
1879 ).toBe(false);
1880 yield* Effect.sync(proxy.release);
1881 const result = yield* Fiber.join(delayed).pipe(Effect.timeout("180 seconds"));
1882 if (Result.isSuccess(result)) {
1883 // A later valid reconcile may win; native leases do not fence LocalState with a global epoch.
1884 yield* assertCommitted(initial.appName, result.success.machineIds);
1885 for (const id of newer.machineIds) {
1886 const retire = proxy.events.findIndex(
1887 (event) =>
1888 event.stage === "request" &&
1889 event.machineId === id &&
1890 (event.path.endsWith("/stop") ||
1891 event.path.endsWith("/cordon") ||
1892 (event.method === "DELETE" && !event.path.endsWith("/lease"))),
1893 );
1894 expect(retire).toBeGreaterThan(0);
1895 for (const candidate of result.success.machineIds) {
1896 expect(
1897 proxy.events
1898 .slice(0, retire)
1899 .some(
1900 (event) =>
1901 event.stage === "completed" &&
1902 event.machineId === candidate &&
1903 event.method === "GET" &&
1904 event.state === "started" &&
1905 event.checks?.some(
1906 (check) => check.name === "ready" && check.status === "passing",
1907 ),
1908 ),
1909 ).toBe(true);
1910 }
1911 }
1912 } else {
1913 expect([
1914 "Fly.DeploymentRecoveryAmbiguous",
1915 "Fly.ReplicaOwnershipChanged",
1916 "Fly.MachineLeaseBusy",
1917 "Fly.MachineLeaseLost",
1918 "NotFound",
1919 ]).toContain(result.failure._tag);
1920 expect(
1921 (yield* census(initial.appName)).some(
1922 (machine) => machine.id === newer.machineId,
1923 ),
1924 ).toBe(true);
1925 yield* Effect.logInfo("Stale snapshot refused without state-fencing guarantee", {
1926 outcome: result.failure._tag,
1927 });
1928 }
1929 const resumed = yield* engineActor(
1930 stack,
1931 "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1932 file,
1933 );
1934 const settled = yield* deployWorker(resumed, "four");
1935 yield* assertCommitted(initial.appName, settled.machineIds);
1936 }).pipe(
1937 Effect.ensuring(
1938 Effect.sync(() => {
1939 proxy.clear();
1940 proxy.release();
1941 }),
1942 ),
1943 Effect.scoped,
1944 );
1945 yield* stack.destroy();
1946 yield* assertAppGone(initial.appName);
1947 }).pipe(Effect.scoped),
1948 { timeout: 600_000 },
1949 );
1950
1951 describe.sequential("independent engine contexts", () => {
1952 for (const competitor of ["bluegreen", "rolling", "destroy"] as const) {
1953 test.provider(
1954 `F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1955 (stack) =>
1956 Effect.gen(function* () {
1957 yield* stack.destroy();
1958 const initial = yield* deployWorker(stack, "one");
1959 const holderProxy = yield* transportProxy();
1960 const contenderProxy = yield* transportProxy();
1961 yield* Effect.sync(() =>
1962 holderProxy.arm({
1963 match: (event) => event.method === "POST" && event.path.endsWith("/machines"),
1964 action: "hold-response",
1965 remaining: 1,
1966 }),
1967 );
1968 yield* Effect.gen(function* () {
1969 const holderActor = yield* engineActor(
1970 stack,
1971 `F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1972 file,
1973 holderProxy.url,
1974 );
1975 const contenderActor = yield* engineActor(
1976 stack,
1977 `F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1978 file,
1979 contenderProxy.url,
1980 );
1981 expect(contenderActor.state).not.toBe(holderActor.state);
1982 const holder = yield* deployWorker(holderActor, "two").pipe(
1983 Effect.scoped,
1984 Effect.forkScoped,
1985 );
1986 const held = yield* holderProxy.wait(
1987 (event) => event.stage === "held" && event.status! < 300,
1988 );
1989 const operation =
1990 competitor === "destroy"
1991 ? contenderActor.destroy()
1992 : deployWorker(contenderActor, "three", {
1993 deploy: { strategy: competitor, healthTimeout: "30 seconds" },
1994 }).pipe(Effect.asVoid);
1995 const result = yield* operation.pipe(
1996 Effect.scoped,
1997 Effect.timeout("75 seconds"),
1998 Effect.result,
1999 );
2000 yield* Effect.sync(holderProxy.release);
2001 expect(Result.isFailure(result)).toBe(true);
2002 if (Result.isFailure(result))
2003 expect(result.failure).not.toMatchObject({ _tag: "TimeoutError" });
2004 expect(
2005 contenderProxy.events.some(
2006 (event) =>
2007 event.stage === "completed" &&
2008 event.status === 409 &&
2009 event.path.endsWith("/lease") &&
2010 event.machineId === initial.machineId,
2011 ),
2012 ).toBe(true);
2013 expect(
2014 contenderProxy.events.some(
2015 (event) =>
2016 event.stage === "completed" &&
2017 event.status! < 300 &&
2018 event.method !== "GET" &&
2019 !event.path.endsWith("/lease"),
2020 ),
2021 ).toBe(false);
2022 const next = yield* Fiber.join(holder).pipe(Effect.timeout("90 seconds"));
2023 expect(next.machineIds).toEqual([held.machineId]);
2024 expect((yield* census(initial.appName)).map((machine) => machine.id)).toEqual(
2025 next.machineIds,
2026 );
2027 }).pipe(
2028 Effect.ensuring(
2029 Effect.sync(() => {
2030 holderProxy.clear();
2031 holderProxy.release();
2032 }),
2033 ),
2034 Effect.scoped,
2035 );
2036 yield* stack.destroy();
2037 yield* assertAppGone(initial.appName);
2038 }).pipe(Effect.scoped),
2039 { timeout: 300_000 },
2040 );
2041 }
2042
2043 test.provider(
2044 "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
2045 (stack) =>
2046 Effect.gen(function* () {
2047 yield* stack.destroy();
2048 const app = yield* stack.deploy(Fly.App("Site"));
2049 const firstProxy = yield* transportProxy();
2050 const secondProxy = yield* transportProxy();
2051 yield* Effect.sync(() =>
2052 firstProxy.arm({
2053 match: (event) => event.method === "POST" && event.path.endsWith("/machines"),
2054 action: "hold-response",
2055 remaining: 1,
2056 }),
2057 );
2058 yield* Effect.gen(function* () {
2059 const firstActor = yield* engineActor(
2060 stack,
2061 "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
2062 file,
2063 firstProxy.url,
2064 );
2065 const secondActor = yield* engineActor(
2066 stack,
2067 "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
2068 file,
2069 secondProxy.url,
2070 );
2071 const first = yield* deployWorker(firstActor, "one").pipe(
2072 Effect.scoped,
2073 Effect.result,
2074 Effect.forkScoped,
2075 );
2076 const created = yield* firstProxy.wait(
2077 (event) => event.stage === "held" && event.status! < 300,
2078 );
2079 const original = (yield* census(app.appName)).find(
2080 (machine) => machine.id === created.machineId,
2081 );
2082 const owner = original?.config?.metadata;
2083 expect(owner?.["alchemy.fqn"]).toBeTruthy();
2084 expect(owner?.["alchemy.instance"]).toBeTruthy();
2085 expect(
2086 firstProxy.events.some(
2087 (event) =>
2088 event.stage === "completed" &&
2089 event.method === "POST" &&
2090 event.path.endsWith("/lease") &&
2091 event.status! < 300,
2092 ),
2093 ).toBe(false);
2094 const second = yield* deployWorker(secondActor, "two").pipe(
2095 Effect.scoped,
2096 Effect.timeout("90 seconds"),
2097 Effect.result,
2098 );
2099 const during = yield* census(app.appName);
2100 yield* Effect.sync(firstProxy.release);
2101 const firstResult = yield* Fiber.join(first).pipe(Effect.timeout("90 seconds"));
2102 yield* Effect.logInfo("First-deploy results: no global exclusion promised", {
2103 first: Result.isFailure(firstResult)
2104 ? firstResult.failure._tag
2105 : firstResult._tag,
2106 second: Result.isFailure(second) ? second.failure._tag : second._tag,
2107 });
2108 const successfulIds: string[] = [];
2109 for (const result of [firstResult, second]) {
2110 if (Result.isSuccess(result)) {
2111 expect(result.success.machineIds.length).toBe(1);
2112 successfulIds.push(...result.success.machineIds);
2113 } else {
2114 expect([
2115 "Fly.DeploymentRecoveryAmbiguous",
2116 "Fly.ReplicaOwnershipChanged",
2117 "Fly.MachineLeaseBusy",
2118 "Fly.MachineLeaseLost",
2119 "Fly.MachineNotCreated",
2120 "NotFound",
2121 ]).toContain(result.failure._tag);
2122 }
2123 }
2124 expect(successfulIds.length).toBeGreaterThan(0);
2125 // A held create response conveys no lease; a leased, ready successor may retire it.
2126 if (Result.isSuccess(second)) {
2127 const committed = yield* assertCommitted(
2128 app.appName,
2129 second.success.machineIds,
2130 );
2131 for (const machine of committed) {
2132 yield* waitHealthy(app.appName, machine, 30_000, machine.config);
2133 }
2134 expect(during.map((machine) => machine.id).sort()).toEqual(
2135 [...second.success.machineIds].sort(),
2136 );
2137 expect(
2138 during.every(
2139 (machine) =>
2140 machine.state === "started" &&
2141 machine.cordoned === false &&
2142 machine.config?.metadata?.["alchemy.phase"] === "active",
2143 ),
2144 ).toBe(true);
2145 } else {
2146 expect(during.some((machine) => machine.id === created.machineId)).toBe(true);
2147 }
2148 const createdIds = new Set(
2149 [firstProxy, secondProxy].flatMap((proxy) =>
2150 proxy.events
2151 .filter(
2152 (event) =>
2153 event.stage === "completed" &&
2154 event.method === "POST" &&
2155 event.path.endsWith("/machines") &&
2156 event.status! < 300,
2157 )
2158 .map((event) => event.machineId),
2159 ),
2160 );
2161 for (const proxy of [firstProxy, secondProxy]) {
2162 for (const [index, event] of proxy.events.entries()) {
2163 if (
2164 event.stage !== "request" ||
2165 !(
2166 event.path.endsWith("/stop") ||
2167 event.path.endsWith("/cordon") ||
2168 event.phase === "retiring" ||
2169 (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path))
2170 )
2171 )
2172 continue;
2173 expect(event.machineId).toBeDefined();
2174 expect(createdIds.has(event.machineId)).toBe(true);
2175 const before = proxy.events.slice(0, index);
2176 const lease = before.findIndex(
2177 (prior) =>
2178 prior.stage === "completed" &&
2179 prior.machineId === event.machineId &&
2180 prior.method === "POST" &&
2181 prior.path.endsWith("/lease") &&
2182 prior.status! < 300,
2183 );
2184 expect(lease).toBeGreaterThan(-1);
2185 expect(
2186 before
2187 .slice(lease + 1)
2188 .some(
2189 (prior) =>
2190 prior.stage === "completed" &&
2191 prior.machineId === event.machineId &&
2192 prior.method === "GET" &&
2193 /\/machines\/[^/]+$/.test(prior.path) &&
2194 prior.status === 200,
2195 ),
2196 ).toBe(true);
2197 expect(
2198 before.some(
2199 (ready, readyIndex) =>
2200 ready.stage === "completed" &&
2201 ready.method === "GET" &&
2202 ready.status === 200 &&
2203 ready.machineId !== event.machineId &&
2204 createdIds.has(ready.machineId) &&
2205 ready.state === "started" &&
2206 ready.cordoned === false &&
2207 ready.checks?.some(
2208 (check) => check.name === "ready" && check.status === "passing",
2209 ) &&
2210 before
2211 .slice(readyIndex + 1)
2212 .some(
2213 (active) =>
2214 active.stage === "completed" &&
2215 active.machineId === ready.machineId &&
2216 active.method === "PATCH" &&
2217 active.phase === "active" &&
2218 active.status! < 300,
2219 ),
2220 ),
2221 ).toBe(true);
2222 }
2223 }
2224 const live = yield* census(app.appName);
2225 expect(live.length).toBe(1);
2226 expect(
2227 live.every(
2228 (machine) =>
2229 successfulIds.includes(machine.id!) &&
2230 machine.state === "started" &&
2231 machine.cordoned === false &&
2232 machine.checks?.some(
2233 (check) => check.name === "ready" && check.status === "passing",
2234 ) &&
2235 machine.config?.metadata?.["alchemy.fqn"] === owner?.["alchemy.fqn"] &&
2236 machine.config?.metadata?.["alchemy.instance"] ===
2237 owner?.["alchemy.instance"],
2238 ),
2239 ).toBe(true);
2240 yield* assertCommitted(
2241 app.appName,
2242 live.map((machine) => machine.id!),
2243 );
2244 }).pipe(
2245 Effect.ensuring(
2246 Effect.sync(() => {
2247 firstProxy.clear();
2248 firstProxy.release();
2249 }),
2250 ),
2251 Effect.scoped,
2252 );
2253 yield* stack.destroy();
2254 yield* assertAppGone(app.appName);
2255 }).pipe(Effect.scoped),
2256 { timeout: 300_000 },
2257 );
2258 });
2259 },
2260 );
2261
2262 describe.sequential(
2263 "commit recovery",
2264 {
2265 tags: [
2266 "provider:fly",
2267 "provider:fly:app",
2268 "provider:fly:ipassignment",
2269 "provider:fly:machine",
2270 "provider:fly:secret",
2271 "provider:fly:service",
2272 "live",
2273 ],
2274 },
2275 () => {
2276 const file = "test/Fly/BlueGreen.test.ts";
2277 const { test } = Test.make({ providers: Fly.providers() });
2278
2279 for (const interrupted of [false, true]) {
2280 test.provider(
2281 `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
2282 (stack) =>
2283 Effect.gen(function* () {
2284 yield* stack.destroy();
2285 const readiness = yield* makeReadinessControl();
2286 const site = yield* readiness.deployApp(stack);
2287 const proxy = yield* transportProxy();
2288 yield* Effect.sync(() =>
2289 proxy.arm({
2290 match: (event) =>
2291 event.path.endsWith("/metadata") && event.phase === "validating",
2292 action: "hold-response",
2293 remaining: 1,
2294 }),
2295 );
2296 const actor = yield* engineActor(
2297 stack,
2298 `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
2299 file,
2300 proxy.url,
2301 );
2302 const attempt = yield* readiness
2303 .deployWorker(actor, "one")
2304 .pipe(Effect.scoped, Effect.result, Effect.forkScoped);
2305 const barrier = yield* proxy
2306 .wait(
2307 (event) =>
2308 event.stage === "held" &&
2309 event.phase === "validating" &&
2310 event.status! >= 200 &&
2311 event.status! < 300,
2312 )
2313 .pipe(
2314 Effect.raceFirst(
2315 Effect.gen(function* () {
2316 const result = yield* Fiber.join(attempt);
2317 if (Result.isFailure(result)) return yield* Effect.fail(result.failure);
2318 return yield* Effect.fail(
2319 new Error("Deployment completed without reaching the validating barrier"),
2320 );
2321 }),
2322 ),
2323 );
2324 const machineId = barrier.machineId!;
2325 expect(machineId).toBeDefined();
2326
2327 const firstRouting = proxy.events.findIndex(
2328 (event) => event.stage === "request" && event.path.endsWith("/uncordon"),
2329 );
2330 expect(firstRouting).toBeGreaterThan(0);
2331 const preparation = proxy.events.slice(0, firstRouting);
2332 const promoting = preparation.findLastIndex(
2333 (event) =>
2334 event.stage === "completed" &&
2335 event.phase === "promoting" &&
2336 event.path.endsWith("/metadata") &&
2337 event.status! < 300,
2338 );
2339 const ready = preparation.findLastIndex(
2340 (event) =>
2341 event.stage === "completed" &&
2342 event.method === "GET" &&
2343 event.path.endsWith(`/machines/${machineId}`) &&
2344 event.state === "started" &&
2345 event.cordoned === true &&
2346 event.checks?.find((check) => check.name === "ready")?.status === "passing",
2347 );
2348 expect(promoting).toBeGreaterThanOrEqual(0);
2349 expect(ready).toBeGreaterThan(promoting);
2350
2351 yield* readiness.turnOff(site.appName, machineId);
2352 if (interrupted) {
2353 const interruption = yield* Fiber.interrupt(attempt).pipe(Effect.forkScoped);
2354 yield* Effect.yieldNow;
2355 yield* Effect.sync(proxy.release);
2356 yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds"));
2357 expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true);
2358 } else {
2359 yield* Effect.sync(proxy.release);
2360 const result = yield* Fiber.join(attempt).pipe(Effect.timeout("180 seconds"));
2361 expect(Result.isFailure(result)).toBe(true);
2362 if (Result.isFailure(result))
2363 expect(result.failure._tag).toBe("Fly.ReplicaChecksNotPassing");
2364 }
2365 yield* Effect.sync(proxy.clear);
2366 const pending = yield* census(site.appName);
2367 expect(pending.map((machine) => machine.id)).toEqual([machineId]);
2368 expect(pending[0]!.cordoned).toBe(false);
2369 const metadata = pending[0]!.config!.metadata!;
2370 expect(metadata["alchemy.phase"]).toBe("validating");
2371 expect(metadata["alchemy.checked-instance"]).toBeUndefined();
2372 const read = yield* observeReplicaSet({
2373 appName: site.appName,
2374 id: "Worker",
2375 type: "Fly.Machine",
2376 fqn: metadata["alchemy.fqn"]!,
2377 resourceInstanceId: metadata["alchemy.instance"]!,
2378 machineIds: [machineId],
2379 });
2380 expect(read?.rolloutPending).toBe(true);
2381 expect(read?.machineIds).toEqual([]);
2382
2383 const retryActor = yield* engineActor(
2384 stack,
2385 `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
2386 file,
2387 proxy.url,
2388 );
2389 const stillBroken = yield* readiness
2390 .deployWorker(retryActor, "one")
2391 .pipe(Effect.scoped, Effect.result);
2392 expect(Result.isFailure(stillBroken)).toBe(true);
2393 if (Result.isFailure(stillBroken))
2394 expect(stillBroken.failure._tag).toBe("Fly.ReplicaChecksNotPassing");
2395 expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual([
2396 machineId,
2397 ]);
2398 expect(
2399 proxy.events.some(
2400 (event) => event.phase === "active" && event.path.endsWith("/metadata"),
2401 ),
2402 ).toBe(false);
2403
2404 yield* repairReadiness(site.appName, machineId);
2405 const recoveryActor = yield* engineActor(
2406 stack,
2407 `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
2408 file,
2409 proxy.url,
2410 );
2411 const recovered = yield* readiness
2412 .deployWorker(recoveryActor, "one")
2413 .pipe(Effect.scoped);
2414 expect(recovered.machineIds).toEqual([machineId]);
2415 yield* assertCommitted(site.appName, recovered.machineIds);
2416 const committed = yield* observeReplicaSet({
2417 appName: site.appName,
2418 id: "Worker",
2419 type: "Fly.Machine",
2420 fqn: metadata["alchemy.fqn"]!,
2421 resourceInstanceId: metadata["alchemy.instance"]!,
2422 machineIds: [machineId],
2423 });
2424 expect(committed?.rolloutPending).toBe(false);
2425 expect(committed?.machineIds).toEqual([machineId]);
2426 yield* stack.destroy();
2427 yield* assertAppGone(site.appName);
2428 }).pipe(Effect.scoped),
2429 { timeout: 600_000 },
2430 );
2431 }
2432 },
2433 );
2434
2435 describe.sequential(
2436 "health",
2437 {
2438 tags: [
2439 "provider:fly",
2440 "provider:fly:app",
2441 "provider:fly:machine",
2442 "provider:fly:service",
2443 "live",
2444 ],
2445 },
2446 () => {
2447 const { test } = Test.make({ providers: Fly.providers() });
2448
2449 for (const kind of ["named", "multiple published services"] as const) {
2450 test.provider(
2451 `S03 every ${kind} check must pass before blue retirement`,
2452 (stack) =>
2453 Effect.gen(function* () {
2454 yield* stack.destroy();
2455 const initial = yield* deployWorker(stack, "one", {
2456 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2457 });
2458 const extra = { ...checks.ready, path: "/does-not-exist" };
2459 const failed = yield* deployWorker(stack, "two", {
2460 checks: kind === "named" ? { ready: checks.ready, dependency: extra } : checks,
2461 services:
2462 kind === "named"
2463 ? undefined
2464 : [
2465 {
2466 protocol: "tcp",
2467 internalPort: 80,
2468 ports: [{ port: 80, handlers: ["http"] }],
2469 checks: [checks.ready],
2470 },
2471 {
2472 protocol: "tcp",
2473 internalPort: 80,
2474 ports: [{ port: 443, handlers: ["tls", "http"] }],
2475 checks: [extra],
2476 },
2477 ],
2478 deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
2479 }).pipe(Effect.result);
2480 expect(Result.isFailure(failed)).toBe(true);
2481 if (Result.isFailure(failed))
2482 expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
2483 const live = yield* census(initial.appName);
2484 expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
2485 expect(live[0]!.cordoned).toBe(false);
2486 const fixed = yield* deployWorker(stack, "two", {
2487 checks: { ready: checks.ready, dependency: checks.ready },
2488 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2489 });
2490 const committed = yield* assertCommitted(initial.appName, fixed.machineIds);
2491 // Active commit metadata can reset reports after readiness validation.
2492 const healthy = yield* waitHealthy(initial.appName, committed[0]!, 30_000);
2493 expect(healthy.instance_id).toBe(committed[0]!.instance_id);
2494 expect(
2495 ["ready", "dependency"].every((name) =>
2496 healthy.checks?.some(
2497 (check) => check.name === name && check.status === "passing",
2498 ),
2499 ),
2500 ).toBe(true);
2501 yield* stack.destroy();
2502 yield* assertAppGone(initial.appName);
2503 }),
2504 { timeout: 300_000 },
2505 );
2506 }
2507 },
2508 );
2509
2510 describe.sequential(
2511 "required metadata writes",
2512 {
2513 tags: [
2514 "provider:fly",
2515 "provider:fly:app",
2516 "provider:fly:machine",
2517 "provider:fly:service",
2518 "live",
2519 ],
2520 },
2521 () => {
2522 let endpoint: string | undefined;
2523 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
2524
2525 const props = {
2526 count: 2,
2527 deploy: { strategy: "bluegreen", healthTimeout: "90 seconds" },
2528 } as const;
2529
2530 test.provider(
2531 "F05 real required promotion-intent write fault forbids every uncordon and early old deletion",
2532 (stack) =>
2533 Effect.gen(function* () {
2534 yield* stack.destroy();
2535 const initial = yield* deployWorker(stack, "one", props);
2536 const proxy = yield* transportProxy();
2537 try {
2538 yield* Effect.sync(() => {
2539 endpoint = proxy.url;
2540 proxy.arm({
2541 match: (event) =>
2542 event.method === "PATCH" &&
2543 event.path.endsWith("/metadata") &&
2544 event.phase === "promoting",
2545 action: "cut-request",
2546 remaining: Infinity,
2547 });
2548 });
2549 const result = yield* deployWorker(stack, "two", props).pipe(Effect.result);
2550 expect(Result.isFailure(result)).toBe(true);
2551 if (Result.isFailure(result))
2552 expect(result.failure).toMatchObject({ _tag: "Fly.MachineMutationUncertain" });
2553 const cut = proxy.events.filter(
2554 (event) => event.stage === "cut" && event.phase === "promoting",
2555 );
2556 expect(cut.length).toBeGreaterThan(0);
2557 expect(new Set(cut.map((event) => event.machineId)).size).toBe(1);
2558 const live = yield* census(initial.appName);
2559 const old = live.filter((machine) => initial.machineIds.includes(machine.id!));
2560 const candidates = live.filter(
2561 (machine) => !initial.machineIds.includes(machine.id!),
2562 );
2563 expect(old.map((machine) => machine.id).sort()).toEqual(
2564 [...initial.machineIds].sort(),
2565 );
2566 expect(
2567 old.every((machine) => machine.cordoned === false && machine.state === "started"),
2568 ).toBe(true);
2569 expect(candidates).toHaveLength(2);
2570 expect(
2571 candidates.every(
2572 (machine) =>
2573 machine.cordoned === true &&
2574 machine.config?.metadata?.["alchemy.phase"] === "candidate",
2575 ),
2576 ).toBe(true);
2577 expect(candidates.some((machine) => machine.id === cut[0]!.machineId)).toBe(true);
2578 expect(proxy.events.some((event) => event.path.endsWith("/uncordon"))).toBe(false);
2579 expect(
2580 proxy.events.some(
2581 (event) =>
2582 initial.machineIds.includes(event.machineId!) &&
2583 (event.path.endsWith("/cordon") ||
2584 event.path.endsWith("/stop") ||
2585 (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path))),
2586 ),
2587 ).toBe(false);
2588 expect(
2589 proxy.events.some(
2590 (event) => event.phase === "active" || event.phase === "retiring",
2591 ),
2592 ).toBe(false);
2593 expect(
2594 proxy.events.some(
2595 (event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path),
2596 ),
2597 ).toBe(false);
2598 const candidateIds = candidates.map((machine) => machine.id!).sort();
2599 yield* Effect.sync(proxy.clear);
2600 const recovered = yield* deployWorker(stack, "two", props);
2601 expect([...recovered.machineIds].sort()).toEqual(candidateIds);
2602 yield* assertCommitted(initial.appName, recovered.machineIds);
2603 } finally {
2604 yield* Effect.sync(() => {
2605 endpoint = undefined;
2606 proxy.clear();
2607 });
2608 }
2609 yield* stack.destroy();
2610 yield* assertAppGone(initial.appName);
2611 }).pipe(
2612 Effect.scoped,
2613 Effect.ensuring(
2614 Effect.sync(() => {
2615 endpoint = undefined;
2616 }),
2617 ),
2618 ),
2619 { timeout: 900_000 },
2620 );
2621 },
2622 );
2623
2624 describe.sequential(
2625 "health transport faults",
2626 {
2627 tags: [
2628 "provider:fly",
2629 "provider:fly:app",
2630 "provider:fly:machine",
2631 "provider:fly:service",
2632 "live",
2633 ],
2634 },
2635 () => {
2636 let endpoint: string | undefined;
2637 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
2638
2639 test.provider(
2640 "S07 health polling recovers a lost real GET and exhausts a persistent connection cut",
2641 (stack) =>
2642 Effect.gen(function* () {
2643 yield* stack.destroy();
2644 const initial = yield* deployWorker(stack, "one", {
2645 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2646 });
2647 const observed = yield* machines.getMachine({
2648 app_name: initial.appName,
2649 machine_id: initial.machineId,
2650 });
2651 const proxy = yield* transportProxy();
2652 const match = (event: { method: string; path: string }) =>
2653 event.method === "GET" && event.path.endsWith(`/machines/${initial.machineId}`);
2654 yield* Effect.sync(() => {
2655 endpoint = proxy.url;
2656 proxy.arm({ match, action: "drop-response", remaining: 1 });
2657 });
2658 try {
2659 const healthy = yield* waitHealthy(
2660 initial.appName,
2661 observed,
2662 30_000,
2663 observed.config!,
2664 );
2665 expect(healthy.id).toBe(initial.machineId);
2666 expect(healthy.instance_id).toBe(observed.instance_id);
2667 expect(healthy.checks?.every((check) => check.status === "passing")).toBe(true);
2668 expect(
2669 proxy.events.filter((event) => event.stage === "dropped" && event.status === 200),
2670 ).toHaveLength(1);
2671 yield* Effect.sync(() =>
2672 proxy.arm({ match, action: "cut-request", remaining: Infinity }),
2673 );
2674 let attempts = 0;
2675 const client = yield* HttpClient.HttpClient;
2676 const observedClient = HttpClient.tapRequest(client, (request) =>
2677 Effect.sync(() => {
2678 if (
2679 request.method === "GET" &&
2680 request.url.endsWith(`/machines/${initial.machineId}`)
2681 ) {
2682 attempts++;
2683 }
2684 }),
2685 );
2686 const started = yield* Clock.currentTimeMillis;
2687 const failed = yield* waitHealthy(
2688 initial.appName,
2689 healthy,
2690 8_000,
2691 healthy.config!,
2692 ).pipe(Effect.provideService(HttpClient.HttpClient, observedClient), Effect.result);
2693 expect((yield* Clock.currentTimeMillis) - started).toBeLessThanOrEqual(12_000);
2694 expect(Result.isFailure(failed)).toBe(true);
2695 if (Result.isFailure(failed)) {
2696 expect(failed.failure).toMatchObject({
2697 _tag: "Fly.ReplicaChecksNotPassing",
2698 appName: initial.appName,
2699 machineId: initial.machineId,
2700 });
2701 }
2702 expect(
2703 proxy.events.filter((event) => event.stage === "cut").length,
2704 ).toBeGreaterThan(0);
2705 // Count logical client calls independently of Bun's physical GET retries.
2706 expect(attempts).toBeGreaterThan(0);
2707 expect(attempts).toBeLessThanOrEqual(11);
2708 const live = yield* census(initial.appName);
2709 expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
2710 expect(live[0]!.cordoned).toBe(false);
2711 } finally {
2712 yield* Effect.sync(() => {
2713 endpoint = undefined;
2714 proxy.clear();
2715 });
2716 }
2717 yield* stack.destroy();
2718 yield* assertAppGone(initial.appName);
2719 }).pipe(
2720 Effect.scoped,
2721 Effect.ensuring(
2722 Effect.sync(() => {
2723 endpoint = undefined;
2724 }),
2725 ),
2726 ),
2727 { timeout: 180_000 },
2728 );
2729 },
2730 );
2731
2732 describe.sequential(
2733 "failures",
2734 {
2735 tags: [
2736 "provider:fly",
2737 "provider:fly:app",
2738 "provider:fly:machine",
2739 "provider:fly:service",
2740 "live",
2741 ],
2742 },
2743 () => {
2744 const { test } = Test.make({ providers: Fly.providers() });
2745
2746 test.provider(
2747 "F01 a rejected candidate create preserves the active generation",
2748 (stack) =>
2749 Effect.gen(function* () {
2750 yield* stack.destroy();
2751 const deploy = (image: string) =>
2752 stack.deploy(
2753 Effect.gen(function* () {
2754 const app = yield* Fly.App("Site");
2755 return yield* Fly.Machine("Worker", {
2756 app,
2757 image,
2758 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2759 shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
2760 checks: {
2761 ready: { type: "http", port: 80, path: "/", interval: "2s", timeout: "1s" },
2762 },
2763 });
2764 }),
2765 );
2766 const initial = yield* deploy("nginx:alpine");
2767 const failed = yield* deploy("nginx:alchemy-bluegreen-nonexistent-image").pipe(
2768 Effect.result,
2769 );
2770 expect(Result.isFailure(failed)).toBe(true);
2771 if (Result.isFailure(failed))
2772 expect(failed.failure).toMatchObject({ _tag: "BadRequest" });
2773 const live = (yield* machines.listMachines({ app_name: initial.appName })).filter(
2774 (machine) => machine.state !== "destroyed",
2775 );
2776 expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
2777 expect(live[0]?.state).toBe("started");
2778 expect(live[0]?.cordoned).toBe(false);
2779 expect(live[0]?.config?.metadata?.["alchemy.phase"]).toBe("active");
2780 yield* stack.destroy();
2781 yield* assertAppGone(initial.appName);
2782 }),
2783 { timeout: 180_000 },
2784 );
2785 },
2786 );
2787 });
2788 describe.sequential("shard 3", () => {
2789 describe.sequential(
2790 "managed HTTP services",
2791 {
2792 tags: [
2793 "provider:fly",
2794 "provider:fly:app",
2795 "provider:fly:ipassignment",
2796 "provider:fly:machine",
2797 "provider:fly:redis",
2798 "provider:fly:secret",
2799 "provider:fly:service",
2800 "live",
2801 ],
2802 },
2803 () => {
2804 const { test } = Test.make({ providers: Fly.providers() });
2805
2806 describe.sequential("Fly live HTTP and worker drain", () => {
2807 for (const policy of [
2808 {
2809 name: "managed old60/new10 SIGTERM",
2810 raw: false,
2811 old: "60 seconds",
2812 next: "10 seconds",
2813 signal: "SIGTERM",
2814 nextSignal: "SIGINT",
2815 rawSignal: undefined,
2816 nextRawSignal: undefined,
2817 delay: 12_000,
2818 },
2819 {
2820 name: "managed old10/new60 SIGINT",
2821 raw: false,
2822 old: "10 seconds",
2823 next: "60 seconds",
2824 signal: "SIGINT",
2825 nextSignal: "SIGTERM",
2826 rawSignal: undefined,
2827 nextRawSignal: undefined,
2828 delay: 3_000,
2829 },
2830 {
2831 name: "raw old60/new10 SIGQUIT to SIGTERM",
2832 raw: true,
2833 old: "60 seconds",
2834 next: "10 seconds",
2835 signal: "SIGTERM",
2836 nextSignal: "SIGTERM",
2837 rawSignal: "SIGQUIT",
2838 nextRawSignal: "SIGTERM",
2839 delay: 12_000,
2840 },
2841 {
2842 name: "raw old10/new60 SIGTERM to SIGQUIT",
2843 raw: true,
2844 old: "10 seconds",
2845 next: "60 seconds",
2846 signal: "SIGTERM",
2847 nextSignal: "SIGTERM",
2848 rawSignal: "SIGTERM",
2849 nextRawSignal: "SIGQUIT",
2850 delay: 3_000,
2851 },
2852 ] as const) {
2853 test.provider(
2854 `R01/R02/R03/R04 ${policy.name} preserves traffic, jobs and streaming after the actual stop barrier`,
2855 (stack) =>
2856 Effect.gen(function* () {
2857 yield* stack.destroy();
2858 const scenario = yield* makeScenario(stack);
2859 yield* Effect.gen(function* () {
2860 const first = yield* scenario.deploy({
2861 raw: policy.raw,
2862 version: "one",
2863 timeout: policy.old,
2864 signal: policy.signal,
2865 rawSignal: policy.rawSignal,
2866 afterSignalMs: policy.delay,
2867 });
2868 const oldWorker = yield* requireValue(first.worker, "initial worker output");
2869 const oldId = oldWorker.machineId;
2870 const url = `https://${first.workerApp.appName}.fly.dev`;
2871 const get = (route: string) =>
2872 HttpClient.get(`${url}${route}`, { headers: { connection: "close" } }).pipe(
2873 Effect.flatMap((response) =>
2874 response.status === 200
2875 ? response.text
2876 : Effect.fail(new Error(`HTTP ${response.status}`)),
2877 ),
2878 Effect.timeout(route === "/" ? "5 seconds" : "240 seconds"),
2879 );
2880 expect(
2881 JSON.parse(
2882 yield* get("/").pipe(
2883 Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") }),
2884 ),
2885 ),
2886 ).toEqual({ machine: oldId, version: "one" });
2887 const observed = yield* machines.getMachine({
2888 app_name: first.workerApp.appName,
2889 machine_id: oldId,
2890 });
2891 expect(observed.config?.stop_config?.signal).toBe(
2892 policy.rawSignal ?? policy.signal,
2893 );
2894 expect(
2895 policy.old === "60 seconds"
2896 ? ["60000ms", "60s", "1m", "1m0s"]
2897 : ["10000ms", "10s"],
2898 ).toContain(observed.config?.stop_config?.timeout);
2899 yield* scenario.call(first.ledgerUrl, "enqueue", ["in-flight-job", "hold"]);
2900 yield* scenario.wait(first.ledgerUrl, (ledger) =>
2901 ledger.events.some(
2902 (event) =>
2903 event.machine === oldId &&
2904 event.event === "claimed" &&
2905 event.job === "in-flight-job",
2906 ),
2907 );
2908 const slow = yield* get("/hold").pipe(Effect.forkScoped);
2909 const streamed = yield* get("/stream").pipe(Effect.forkScoped);
2910 yield* scenario.wait(
2911 first.ledgerUrl,
2912 (ledger) =>
2913 ledger.events.filter(
2914 (event) => event.machine === oldId && event.event === "request-started",
2915 ).length === 2,
2916 );
2917 const responses = yield* Ref.make<
2918 Array<{ machine: string; version: string } | { failure: string }>
2919 >([]);
2920 const finished = yield* Ref.make(false);
2921 yield* Effect.addFinalizer(() =>
2922 Ref.get(responses).pipe(
2923 Effect.flatMap((samples) =>
2924 Effect.logInfo("Unretried public traffic samples", samples),
2925 ),
2926 ),
2927 );
2928 const traffic = yield* Stream.range(0, 599).pipe(
2929 Stream.mapEffect(() =>
2930 get("/").pipe(
2931 Effect.result,
2932 Effect.flatMap((result) =>
2933 Ref.update(responses, (values) => [
2934 ...values,
2935 Result.isSuccess(result)
2936 ? (JSON.parse(result.success) as {
2937 machine: string;
2938 version: string;
2939 })
2940 : { failure: String(result.failure) },
2941 ]),
2942 ),
2943 Effect.andThen(Effect.sleep("500 millis")),
2944 Effect.andThen(Ref.get(finished)),
2945 ),
2946 ),
2947 Stream.takeUntil((done) => done),
2948 Stream.runDrain,
2949 Effect.forkScoped,
2950 );
2951 const second = yield* scenario.deploy({
2952 raw: policy.raw,
2953 version: "two",
2954 timeout: policy.next,
2955 signal: policy.nextSignal,
2956 rawSignal: policy.nextRawSignal,
2957 afterSignalMs: 1000,
2958 });
2959 const newWorker = yield* requireValue(
2960 second.worker,
2961 "replacement worker output",
2962 );
2963 expect(JSON.parse(yield* Fiber.join(slow))).toEqual({
2964 machine: oldId,
2965 version: "one",
2966 });
2967 // The fixture pads quiet gaps with blank heartbeat lines so Fly's proxy
2968 // never sees a 60s-idle connection; the payload itself must be intact.
2969 const body = (yield* Fiber.join(streamed)).replace(/^\n+/gm, "");
2970 const expected = "first\n".repeat(32768) + "last\n".repeat(32768);
2971 const hashes = yield* Effect.sync(() =>
2972 [body, expected].map((value) =>
2973 createHash("sha256").update(value).digest("hex"),
2974 ),
2975 );
2976 expect(body.length).toBe(360448);
2977 expect(hashes[0]).toBe(hashes[1]);
2978 expect(JSON.parse(yield* get("/"))).toEqual({
2979 machine: newWorker.machineId,
2980 version: "two",
2981 });
2982 yield* Effect.sleep("600 millis");
2983 yield* Ref.set(finished, true);
2984 yield* Fiber.join(traffic);
2985 const samples = yield* Ref.get(responses);
2986 expect(samples.filter((sample) => "failure" in sample)).toEqual([]);
2987 expect(
2988 samples.some(
2989 (sample) =>
2990 "version" in sample &&
2991 sample.version === "one" &&
2992 sample.machine === oldId,
2993 ),
2994 ).toBe(true);
2995 expect(
2996 samples.some(
2997 (sample) =>
2998 "version" in sample &&
2999 sample.version === "two" &&
3000 sample.machine === newWorker.machineId,
3001 ),
3002 ).toBe(true);
3003 yield* assertReplacement(first.workerApp.appName, oldId, newWorker.machineId);
3004 const ledger = yield* scenario.settle(first.ledgerUrl);
3005 assertStopped(ledger.events, oldId);
3006 assertOrder(ledger.events, oldId, "request-finalized", "shared-closed");
3007 assertOrder(ledger.events, oldId, "client-released", "shared-closed");
3008 const stop = yield* requireValue(
3009 ledger.events.find(
3010 (event) => event.machine === oldId && event.event === "stop-started",
3011 ),
3012 "old worker stop event",
3013 );
3014 if (policy.raw) expect(stop.signal).toBe(policy.rawSignal);
3015 const completed = ledger.events.filter(
3016 (event) => event.machine === oldId && event.event === "response-finished",
3017 );
3018 expect(completed).toHaveLength(2);
3019 for (const event of completed) {
3020 expect(event.at - stop.at).toBeGreaterThan(policy.delay - 500);
3021 }
3022 const ack = yield* requireValue(
3023 ledger.events.find(
3024 (event) =>
3025 event.machine === oldId &&
3026 event.event === "acked" &&
3027 event.job === "in-flight-job",
3028 ),
3029 "in-flight job ACK event",
3030 );
3031 expect(ack.ack).toBe(1);
3032 expect(ack.at - stop.at).toBeGreaterThan(policy.delay - 500);
3033 if (policy.old === "10 seconds") expect(ack.at - stop.at).toBeLessThan(10_000);
3034 }).pipe(Effect.scoped, Effect.ensuring(scenario.cleanup));
3035 }),
3036 {
3037 // Includes cold publication, both readiness phases, proxy overlap, post-signal work and census.
3038 timeout: 480_000,
3039 },
3040 );
3041 }
3042 });
3043 },
3044 );
3045
3046 describe.sequential(
3047 "legacy compatibility",
3048 {
3049 tags: [
3050 "provider:fly",
3051 "provider:fly:app",
3052 "provider:fly:machine",
3053 "provider:fly:service",
3054 "live",
3055 ],
3056 },
3057 () => {
3058 const file = "test/Fly/BlueGreen.test.ts";
3059 let endpoint: string | undefined;
3060 let stops: StopRequest[] = [];
3061 const { test } = Test.make({
3062 providers: observeStops(
3063 () => endpoint,
3064 (request) => stops.push(request),
3065 ),
3066 });
3067
3068 const runningLegacy = (appName: string, id: string) =>
3069 machines.getMachine({ app_name: appName, machine_id: id }).pipe(
3070 Effect.repeat({
3071 schedule: Schedule.spaced("2 seconds"),
3072 times: 45,
3073 until: (machine) =>
3074 machine.state === "started" &&
3075 machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined &&
3076 machine.config?.metadata?.["alchemy.generation"] === undefined &&
3077 machine.config?.stop_config === undefined,
3078 }),
3079 Effect.timeout("120 seconds"),
3080 Effect.tap((machine) =>
3081 Effect.sync(() => {
3082 expect(machine.state).toBe("started");
3083 expect(machine.config?.stop_config).toBeUndefined();
3084 expect(machine.config?.metadata?.["alchemy.deployment-protocol"]).toBeUndefined();
3085 expect(machine.config?.metadata?.["alchemy.generation"]).toBeUndefined();
3086 }),
3087 ),
3088 );
3089
3090 describe.sequential("live legacy protocol compatibility models", () => {
3091 for (const runtimeTimeoutMs of [undefined, 10_000, 60_000]) {
3092 test.provider(
3093 `F14 live legacy missing stop_config ${runtimeTimeoutMs === undefined ? "preserves raw-image defaults" : `uses ${runtimeTimeoutMs}ms runtime-env fallback`}`,
3094 (stack) =>
3095 Effect.gen(function* () {
3096 yield* stack.destroy();
3097 const site = yield* stack.deploy(Fly.App("Site"));
3098 try {
3099 const initial = yield* deployWorker(stack, "one", {
3100 deploy: { strategy: "rolling" },
3101 shutdown: undefined,
3102 });
3103 yield* writeLegacyProtocol(site.appName, initial.machineId, runtimeTimeoutMs);
3104 const legacy = yield* runningLegacy(site.appName, initial.machineId);
3105 expect(legacy.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS).toBe(
3106 runtimeTimeoutMs?.toString(),
3107 );
3108 const policy = yield* predecessorShutdown(legacy);
3109 if (runtimeTimeoutMs === undefined) {
3110 expect(policy.signal).toBeUndefined();
3111 expect(policy.timeout).toBeUndefined();
3112 } else {
3113 expect(policy.signal).toBe("SIGTERM");
3114 expect(policy.timeout).toBe(`${runtimeTimeoutMs}ms`);
3115 expect(policy.timeoutMs).toBe(runtimeTimeoutMs);
3116 }
3117 const proxy = yield* transportProxy();
3118 yield* Effect.sync(() => {
3119 endpoint = proxy.url;
3120 stops = [];
3121 });
3122 const upgraded = yield* deployWorker(stack, "two", {
3123 deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
3124 });
3125 expect(upgraded.machineId).not.toBe(initial.machineId);
3126 yield* assertCommitted(site.appName, upgraded.machineIds);
3127 const stop = stops.filter((request) => request.machineId === initial.machineId);
3128 expect(stop).toHaveLength(1);
3129 expect(stop[0]!.signal).toBe(
3130 runtimeTimeoutMs === undefined ? undefined : "SIGTERM",
3131 );
3132 expect(stop[0]!.timeout).toBe(
3133 runtimeTimeoutMs === undefined ? undefined : `${runtimeTimeoutMs}ms`,
3134 );
3135 expect(
3136 proxy.events.some(
3137 (event) =>
3138 event.stage === "completed" &&
3139 event.machineId === initial.machineId &&
3140 event.path.endsWith("/stop") &&
3141 event.status! >= 200 &&
3142 event.status! < 300,
3143 ),
3144 ).toBe(true);
3145 expect(
3146 proxy.events.some(
3147 (event) =>
3148 event.stage === "completed" &&
3149 event.machineId === initial.machineId &&
3150 event.method === "DELETE" &&
3151 /\/machines\/[^/]+$/.test(event.path) &&
3152 event.status! >= 200 &&
3153 event.status! < 300,
3154 ),
3155 ).toBe(true);
3156 const same = yield* deployWorker(stack, "two", {
3157 deploy: { strategy: "bluegreen", healthTimeout: "65 seconds" },
3158 });
3159 expect(same.machineIds).toEqual(upgraded.machineIds);
3160 } finally {
3161 yield* Effect.sync(() => {
3162 endpoint = undefined;
3163 stops = [];
3164 });
3165 yield* stack.destroy();
3166 yield* assertAppGone(site.appName);
3167 }
3168 }).pipe(Effect.scoped),
3169 { timeout: 900_000 },
3170 );
3171 }
3172
3173 for (const affected of [1, 2]) {
3174 test.provider(
3175 `F14 unknown metadata protocol on ${affected} of 2 real Machines safely refuses without mutation`,
3176 (stack) =>
3177 Effect.gen(function* () {
3178 yield* stack.destroy();
3179 const site = yield* stack.deploy(Fly.App("Site"));
3180 try {
3181 const initial = yield* deployWorker(stack, "one", { count: 2 });
3182 for (const id of initial.machineIds.slice(0, affected)) {
3183 const target = { app_name: site.appName, machine_id: id };
3184 const current = yield* machines.getMachine(target);
3185 yield* machines.patchMachineMetadata({
3186 ...target,
3187 metadata: {
3188 ...current.config!.metadata,
3189 "alchemy.deployment-protocol": "future-unknown",
3190 },
3191 });
3192 }
3193 const before = yield* census(site.appName);
3194 expect(
3195 before.filter(
3196 (machine) =>
3197 machine.config?.metadata?.["alchemy.deployment-protocol"] ===
3198 "future-unknown",
3199 ),
3200 ).toHaveLength(affected);
3201 const metadata = before[0]!.config!.metadata!;
3202 const read = yield* observeReplicaSet({
3203 appName: site.appName,
3204 id: "Worker",
3205 type: "Fly.Machine",
3206 fqn: metadata["alchemy.fqn"]!,
3207 resourceInstanceId: metadata["alchemy.instance"]!,
3208 machineIds: initial.machineIds,
3209 });
3210 expect(read?.rolloutPending).toBe(true);
3211 expect(read?.machineIds).toEqual([]);
3212 const proxy = yield* transportProxy();
3213 yield* Effect.sync(() => {
3214 endpoint = proxy.url;
3215 });
3216 const result = yield* deployWorker(stack, "two", { count: 2 }).pipe(
3217 Effect.timeout("180 seconds"),
3218 Effect.result,
3219 );
3220 expect(Result.isFailure(result)).toBe(true);
3221 if (Result.isFailure(result)) {
3222 expect(result.failure).toMatchObject({
3223 _tag: "Fly.DeploymentRecoveryAmbiguous",
3224 });
3225 }
3226 expect(
3227 proxy.events.some(
3228 (event) =>
3229 event.stage === "request" &&
3230 event.method !== "GET" &&
3231 !event.path.endsWith("/lease"),
3232 ),
3233 ).toBe(false);
3234 const after = yield* census(site.appName);
3235 expect(after.map((machine) => machine.id).sort()).toEqual(
3236 initial.machineIds.slice().sort(),
3237 );
3238 for (const machine of after) {
3239 const previous = before.find((item) => item.id === machine.id)!;
3240 expect(machine.instance_id).toBe(previous.instance_id);
3241 expect(machine.config).toEqual(previous.config);
3242 expect(machine.cordoned).toBe(false);
3243 expect(machine.state).toBe("started");
3244 }
3245 } finally {
3246 yield* Effect.sync(() => {
3247 endpoint = undefined;
3248 });
3249 yield* stack.destroy();
3250 yield* assertAppGone(site.appName);
3251 }
3252 }).pipe(Effect.scoped),
3253 { timeout: 600_000 },
3254 );
3255 }
3256
3257 test.provider(
3258 "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
3259 (stack) =>
3260 Effect.gen(function* () {
3261 yield* stack.destroy();
3262 const site = yield* stack.deploy(Fly.App("Site"));
3263 try {
3264 const first = yield* deployWorker(stack, "one", { count: 2 });
3265 yield* writeLegacyProtocol(site.appName, first.machineIds[0]!, 60_000);
3266 yield* runningLegacy(site.appName, first.machineIds[0]!);
3267 const mixed = yield* census(site.appName);
3268 expect(
3269 mixed.filter(
3270 (machine) =>
3271 machine.config?.metadata?.["alchemy.deployment-protocol"] === "1",
3272 ),
3273 ).toHaveLength(1);
3274 expect(
3275 mixed.filter(
3276 (machine) =>
3277 machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined,
3278 ),
3279 ).toHaveLength(1);
3280 const proxy = yield* transportProxy();
3281 try {
3282 yield* Effect.sync(() =>
3283 proxy.arm({
3284 match: (event) =>
3285 event.method === "POST" && event.path.endsWith("/machines"),
3286 action: "hold-response",
3287 remaining: 1,
3288 }),
3289 );
3290 yield* Effect.gen(function* () {
3291 const actor = yield* engineActor(
3292 stack,
3293 "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
3294 file,
3295 proxy.url,
3296 );
3297 const upgrade = yield* deployWorker(actor, "two", { count: 2 }).pipe(
3298 Effect.scoped,
3299 Effect.forkScoped,
3300 );
3301 yield* proxy.wait(
3302 (event) =>
3303 event.stage === "held" && event.status! >= 200 && event.status! < 300,
3304 );
3305 const competing = yield* writeLegacyProtocol(
3306 site.appName,
3307 first.machineIds[1]!,
3308 ).pipe(Effect.result);
3309 expect(Result.isFailure(competing)).toBe(true);
3310 if (Result.isFailure(competing))
3311 expect(competing.failure._tag).toBe("Conflict");
3312 expect(
3313 (yield* census(site.appName)).filter((machine) =>
3314 first.machineIds.includes(machine.id!),
3315 ),
3316 ).toHaveLength(2);
3317 yield* Effect.sync(() => {
3318 proxy.clear();
3319 proxy.release();
3320 });
3321 const upgraded = yield* Fiber.join(upgrade).pipe(
3322 Effect.timeout("300 seconds"),
3323 );
3324 yield* assertCommitted(site.appName, upgraded.machineIds);
3325 // A lease-aware legacy writer can still change a successor after lease release.
3326 yield* writeLegacyProtocol(site.appName, upgraded.machineIds[0]!, 60_000);
3327 yield* runningLegacy(site.appName, upgraded.machineIds[0]!);
3328 const partial = yield* census(site.appName);
3329 expect(partial.map((machine) => machine.id).sort()).toEqual(
3330 upgraded.machineIds.slice().sort(),
3331 );
3332 expect(
3333 partial.filter(
3334 (machine) =>
3335 machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined,
3336 ),
3337 ).toHaveLength(1);
3338 const resumed = yield* engineActor(
3339 stack,
3340 "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
3341 file,
3342 );
3343 const recovered = yield* deployWorker(resumed, "three", { count: 2 }).pipe(
3344 Effect.scoped,
3345 );
3346 expect(
3347 recovered.machineIds.every((id) => !upgraded.machineIds.includes(id)),
3348 ).toBe(true);
3349 yield* assertCommitted(site.appName, recovered.machineIds);
3350 yield* Effect.logInfo("Partial-upgrade model boundary", {
3351 model:
3352 "native lease-aware legacy protocol writer, not a literal old binary",
3353 globalFence: false,
3354 recoveredCount: recovered.machineIds.length,
3355 });
3356 }).pipe(
3357 Effect.ensuring(
3358 Effect.sync(() => {
3359 proxy.clear();
3360 proxy.release();
3361 }),
3362 ),
3363 Effect.scoped,
3364 );
3365 } finally {
3366 yield* Effect.sync(() => {
3367 proxy.clear();
3368 proxy.release();
3369 });
3370 }
3371 } finally {
3372 yield* stack.destroy();
3373 yield* assertAppGone(site.appName);
3374 }
3375 }).pipe(Effect.scoped),
3376 { timeout: 900_000 },
3377 );
3378 });
3379 },
3380 );
3381
3382 describe.sequential(
3383 "health deadlines",
3384 {
3385 tags: [
3386 "provider:fly",
3387 "provider:fly:app",
3388 "provider:fly:machine",
3389 "provider:fly:service",
3390 "live",
3391 ],
3392 },
3393 () => {
3394 const { test } = Test.make({ providers: Fly.providers() });
3395
3396 for (const budget of [8_000, 90_000]) {
3397 test.provider(
3398 `S06 real delayed readiness beyond the default minute with ${budget}ms budget`,
3399 (stack) =>
3400 Effect.gen(function* () {
3401 yield* stack.destroy();
3402 const initial = yield* deployWorker(stack, "one");
3403 const result = yield* deployWorker(stack, "two", {
3404 init: { exec: ["/bin/sh", "-c", "sleep 65; exec nginx -g 'daemon off;'"] },
3405 checks: { ready: { ...checks.ready, gracePeriod: "65s" } },
3406 deploy: { strategy: "bluegreen", healthTimeout: budget },
3407 }).pipe(Effect.result);
3408 if (budget === 8_000) {
3409 expect(Result.isFailure(result)).toBe(true);
3410 if (Result.isFailure(result))
3411 expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
3412 const live = yield* census(initial.appName);
3413 expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
3414 expect(live[0]!.cordoned).toBe(false);
3415 } else {
3416 expect(Result.isSuccess(result)).toBe(true);
3417 if (Result.isSuccess(result)) {
3418 expect(result.success.machineId).not.toBe(initial.machineId);
3419 yield* assertCommitted(initial.appName, result.success.machineIds);
3420 }
3421 }
3422 yield* stack.destroy();
3423 yield* assertAppGone(initial.appName);
3424 }),
3425 { timeout: 600_000 },
3426 );
3427 }
3428 },
3429 );
3430
3431 describe.sequential(
3432 "validation",
3433 {
3434 tags: [
3435 "provider:fly",
3436 "provider:fly:app",
3437 "provider:fly:machine",
3438 "provider:fly:service",
3439 "live",
3440 ],
3441 },
3442 () => {
3443 let endpoint: string | undefined;
3444 const { test } = Test.make({ providers: throughProxy(() => endpoint) });
3445 const invalid: Array<[string, Partial<Extract<MachineProps, { image: string }>>]> = [
3446 ["direct volume", { mounts: [{ path: "/data", sizeGb: 1 }] }],
3447 ["skipLaunch", { skipLaunch: true }],
3448 ["autoDestroy", { autoDestroy: true }],
3449 ["restart=no", { restart: { policy: "no" } }],
3450 ["missing checks", { checks: {} }],
3451 ["unchecked public service", { services: [{ internalPort: 80, ports: [{ port: 80 }] }] }],
3452 ];
3453
3454 describe.sequential("pre-mutation validation", () => {
3455 for (const [label, props] of invalid) {
3456 test.provider(
3457 `${label === "direct volume" ? "S10" : "S11"} ${label} rejects before any Machine mutation`,
3458 (stack) =>
3459 Effect.gen(function* () {
3460 yield* stack.destroy();
3461 const app = yield* stack.deploy(Fly.App("Site"));
3462 const proxy = yield* transportProxy();
3463 yield* Effect.sync(() => {
3464 endpoint = proxy.url;
3465 });
3466 const failed = yield* stack
3467 .deploy(
3468 Effect.gen(function* () {
3469 const app = yield* Fly.App("Site");
3470 return yield* Fly.Machine("Worker", {
3471 app,
3472 image: "nginx:alpine",
3473 checks,
3474 deploy: { strategy: "bluegreen" },
3475 ...props,
3476 });
3477 }),
3478 )
3479 .pipe(Effect.result);
3480 expect(Result.isFailure(failed)).toBe(true);
3481 if (Result.isFailure(failed))
3482 expect(failed.failure).toMatchObject({ _tag: "Fly.InvalidDeployment" });
3483 expect(
3484 proxy.events.some(
3485 (event) =>
3486 event.method !== "GET" && /\/(machines|volumes)(\/|$)/.test(event.path),
3487 ),
3488 ).toBe(false);
3489 expect(yield* census(app.appName)).toHaveLength(0);
3490 expect(yield* machines.listVolumes({ app_name: app.appName })).toHaveLength(0);
3491 yield* Effect.sync(() => {
3492 endpoint = undefined;
3493 });
3494 yield* stack.destroy();
3495 yield* assertAppGone(app.appName);
3496 }).pipe(
3497 Effect.scoped,
3498 Effect.ensuring(
3499 Effect.sync(() => {
3500 endpoint = undefined;
3501 }),
3502 ),
3503 ),
3504 { timeout: 180_000 },
3505 );
3506 }
3507
3508 test.provider(
3509 "S10 resolved MountVolume binding rejects before Machine or Volume mutation",
3510 (stack) =>
3511 Effect.gen(function* () {
3512 yield* stack.destroy();
3513 const app = yield* stack.deploy(Fly.App("Site"));
3514 const proxy = yield* transportProxy();
3515 yield* Effect.sync(() => {
3516 endpoint = proxy.url;
3517 });
3518 const failed = yield* stack.deploy(MountedBlueGreen).pipe(Effect.result);
3519 expect(Result.isFailure(failed)).toBe(true);
3520 if (Result.isFailure(failed))
3521 expect(failed.failure).toMatchObject({ _tag: "Fly.InvalidDeployment" });
3522 expect(
3523 proxy.events.some(
3524 (event) =>
3525 event.method !== "GET" && /\/(machines|volumes)(\/|$)/.test(event.path),
3526 ),
3527 ).toBe(false);
3528 expect(yield* census(app.appName)).toHaveLength(0);
3529 expect(yield* machines.listVolumes({ app_name: app.appName })).toHaveLength(0);
3530 yield* Effect.sync(() => {
3531 endpoint = undefined;
3532 });
3533 yield* stack.destroy();
3534 yield* assertAppGone(app.appName);
3535 }).pipe(
3536 Effect.scoped,
3537 Effect.ensuring(
3538 Effect.sync(() => {
3539 endpoint = undefined;
3540 }),
3541 ),
3542 ),
3543 { tags: ["provider:fly:volume"], timeout: 300_000 },
3544 );
3545 });
3546 },
3547 );
3548
3549 describe.sequential(
3550 "runtime secrets",
3551 {
3552 tags: [
3553 "provider:fly",
3554 "provider:fly:app",
3555 "provider:fly:ipassignment",
3556 "provider:fly:machine",
3557 "provider:fly:secret",
3558 "provider:fly:service",
3559 "live",
3560 ],
3561 },
3562 () => {
3563 const { test } = Test.make({ providers: Fly.providers() });
3564
3565 class WriterProbeFailed extends Data.TaggedError("WriterProbeFailed")<{
3566 stage: "transport" | "status" | "decode";
3567 status?: number;
3568 }> {}
3569
3570 const Receipt = Schema.Struct({
3571 version: Schema.Number,
3572 machineId: Schema.String,
3573 marker: Schema.Literals(["ready", "three"]),
3574 });
3575
3576 const probeWriter = (
3577 appName: string,
3578 token: Redacted.Redacted<string>,
3579 method: "GET" | "POST",
3580 ) =>
3581 Effect.gen(function* () {
3582 const client = yield* HttpClient.HttpClient;
3583 const url = `https://${appName}.fly.dev/writer`;
3584 const request = (
3585 method === "POST" ? HttpClientRequest.post(url) : HttpClientRequest.get(url)
3586 ).pipe(HttpClientRequest.bearerToken(token));
3587 const response = yield* client
3588 .execute(request)
3589 .pipe(Effect.mapError(() => new WriterProbeFailed({ stage: "transport" })));
3590 if (response.status !== 200) {
3591 return yield* new WriterProbeFailed({ stage: "status", status: response.status });
3592 }
3593 return yield* response.json.pipe(
3594 Effect.flatMap(Schema.decodeUnknownEffect(Receipt, { onExcessProperty: "error" })),
3595 Effect.mapError(() => new WriterProbeFailed({ stage: "decode" })),
3596 );
3597 }).pipe(
3598 Effect.timeout("30 seconds"),
3599 Effect.catchTag("TimeoutError", () =>
3600 Effect.fail(new WriterProbeFailed({ stage: "transport" })),
3601 ),
3602 );
3603
3604 const marker = (appName: string, machineId: string) =>
3605 machines
3606 .execMachine({
3607 app_name: appName,
3608 machine_id: machineId,
3609 command: ["cat", "/usr/share/nginx/html/marker"],
3610 timeout: 5,
3611 })
3612 .pipe(
3613 Effect.map((response) => {
3614 const value = response.stdout?.trim();
3615 return {
3616 code: response.exit_code,
3617 marker: value === "two" || value === "three" ? value : "missing",
3618 };
3619 }),
3620 );
3621
3622 test.provider(
3623 "F12 deployed sibling runtime writer advances the shared vault during a held candidate create",
3624 (stack) =>
3625 Effect.gen(function* () {
3626 yield* stack.destroy();
3627 const proxy = yield* transportProxy();
3628 let appName: string | undefined;
3629 yield* Effect.addFinalizer(() =>
3630 stack.destroy().pipe(
3631 Effect.andThen(() =>
3632 appName === undefined ? Effect.void : assertAppGone(appName),
3633 ),
3634 Effect.orDie,
3635 ),
3636 );
3637 const actor = yield* engineActor(
3638 stack,
3639 "F12 deployed sibling runtime writer advances the shared vault during a held candidate create",
3640 "test/Fly/BlueGreen.test.ts",
3641 proxy.url,
3642 );
3643 const trigger = yield* Effect.sync(() =>
3644 Redacted.make(
3645 Array.from(randomBytes(32), (byte) => byte.toString(16).padStart(2, "0")).join(
3646 "",
3647 ),
3648 ),
3649 );
3650 const deploy = (count: number, floor?: number) =>
3651 actor.deploy(
3652 Effect.gen(function* () {
3653 const app = yield* Site;
3654 const secret = yield* Token;
3655 const auth = yield* Fly.Secret("WriterTrigger", {
3656 app,
3657 name: TRIGGER_SECRET,
3658 value: trigger,
3659 });
3660 yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
3661 const writer = yield* Writer.pipe(Effect.provide(writerLayer(auth.digest)));
3662 const consumer = yield* Fly.Machine("Consumer", {
3663 app,
3664 region: "iad",
3665 image: "nginx:alpine",
3666 count,
3667 minSecretsVersion: floor,
3668 env: { SECRET_RESOURCE_NAME: secret.name },
3669 init: {
3670 exec: [
3671 "/bin/sh",
3672 "-c",
3673 "case \"$ACCEPTANCE_RUNTIME_SECRET\" in *-two) marker=two;; *-three) marker=three;; *) marker=missing;; esac; printf '%s' \"$marker\" > /usr/share/nginx/html/marker; exec nginx -g 'daemon off;'",
3674 ],
3675 },
3676 checks,
3677 deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
3678 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
3679 });
3680 return { app, secret, writer, consumer };
3681 }),
3682 );
3683
3684 const initial = yield* deploy(1);
3685 appName = initial.app.appName;
3686 expect(initial.writer.machineIds).toHaveLength(1);
3687 expect(initial.consumer.machineIds).toHaveLength(1);
3688 expect(initial.consumer.imageRef?.digest).toBeTruthy();
3689 expect(yield* marker(appName, initial.consumer.machineId)).toEqual({
3690 code: 0,
3691 marker: "two",
3692 });
3693 const before = yield* probeWriter(appName, trigger, "GET").pipe(
3694 Effect.retry({
3695 while: (error) =>
3696 error.stage === "transport" ||
3697 (error.stage === "status" && [404, 502, 503].includes(error.status ?? 0)),
3698 schedule: Schedule.spaced("2 seconds"),
3699 times: 8,
3700 }),
3701 Effect.timeout("90 seconds"),
3702 );
3703 expect(before).toEqual({
3704 version: 0,
3705 machineId: initial.writer.machineId,
3706 marker: "ready",
3707 });
3708 const client = yield* HttpClient.HttpClient;
3709 const unauthorized = yield* client.post(`https://${appName}.fly.dev/writer`).pipe(
3710 Effect.timeout("30 seconds"),
3711 Effect.mapError(() => new WriterProbeFailed({ stage: "transport" })),
3712 );
3713 expect(unauthorized.status).toBe(401);
3714 const initialCensus = yield* census(appName);
3715 expect(initialCensus.map((machine) => machine.id).sort()).toEqual(
3716 [...initial.writer.machineIds, ...initial.consumer.machineIds].sort(),
3717 );
3718 const writerBefore = initialCensus.find(
3719 (machine) => machine.id === initial.writer.machineId,
3720 );
3721 expect(writerBefore?.instance_id).toBeTruthy();
3722 expect(writerBefore?.image_ref?.digest).toBeTruthy();
3723 const secretWrites = proxy.events.filter(
3724 (event) =>
3725 event.stage === "completed" &&
3726 event.method === "POST" &&
3727 event.path.startsWith(`/v1/apps/${appName}/secrets`) &&
3728 event.status !== undefined &&
3729 event.status >= 200 &&
3730 event.status < 300 &&
3731 event.secretsVersion !== undefined,
3732 );
3733 expect(secretWrites.length).toBeGreaterThan(0);
3734 const floor = Math.max(...secretWrites.map((event) => event.secretsVersion!));
3735 expect(Number.isSafeInteger(floor)).toBe(true);
3736 expect(floor).toBeGreaterThan(0);
3737 const start = proxy.events.length;
3738 const machinePath = `/v1/apps/${appName}/machines`;
3739 yield* Effect.sync(() =>
3740 proxy.arm({
3741 match: (event) =>
3742 event.method === "POST" &&
3743 event.path === machinePath &&
3744 event.phase === "candidate",
3745 action: "hold-response",
3746 remaining: 1,
3747 }),
3748 );
3749 yield* Effect.gen(function* () {
3750 const rollout = yield* deploy(2, floor).pipe(Effect.scoped, Effect.forkScoped);
3751 const held = yield* proxy.wait(
3752 (event) =>
3753 event.stage === "held" &&
3754 event.path === machinePath &&
3755 event.status !== undefined &&
3756 event.status >= 200 &&
3757 event.status < 300,
3758 );
3759 expect(held.machineId).toBeTruthy();
3760 expect(held.minSecretsVersion).toBe(floor);
3761 // This POST runs the existing WriteSecret binding inside the sibling Machine.
3762 const later = yield* probeWriter(initial.app.appName, trigger, "POST");
3763 expect(later.machineId).toBe(initial.writer.machineId);
3764 expect(later.marker).toBe("three");
3765 expect(Number.isSafeInteger(later.version)).toBe(true);
3766 expect(later.version).toBeGreaterThan(floor);
3767 expect(
3768 proxy.events.some(
3769 (event) => event.sequence === held.sequence && event.stage === "forwarded",
3770 ),
3771 ).toBe(false);
3772 yield* Effect.sync(proxy.release);
3773 const next = yield* Fiber.join(rollout).pipe(Effect.timeout("240 seconds"));
3774 expect(next.app.appName).toBe(initial.app.appName);
3775 expect(next.secret.name).toBe(initial.secret.name);
3776 expect(next.writer.machineIds).toEqual(initial.writer.machineIds);
3777 expect(next.consumer.machineIds).toHaveLength(2);
3778 expect(next.consumer.machineIds).toContain(held.machineId);
3779 expect(next.consumer.machineIds).not.toContain(initial.consumer.machineId);
3780 const events = proxy.events.slice(start);
3781 const creates = events.filter(
3782 (event) =>
3783 event.stage === "request" &&
3784 event.method === "POST" &&
3785 event.path === machinePath,
3786 );
3787 expect(creates).toHaveLength(2);
3788 expect(creates[0]?.sequence).toBe(held.sequence);
3789 expect(creates.every((event) => event.minSecretsVersion === floor)).toBe(true);
3790 const released = events.findIndex(
3791 (event) => event.sequence === held.sequence && event.stage === "forwarded",
3792 );
3793 expect(released).toBeGreaterThan(-1);
3794 expect(events.indexOf(creates[1]!)).toBeGreaterThan(released);
3795 expect(
3796 events.some(
3797 (event) =>
3798 event.stage === "request" &&
3799 event.machineId === initial.writer.machineId &&
3800 event.method !== "GET",
3801 ),
3802 ).toBe(false);
3803 const live = yield* census(initial.app.appName);
3804 expect(live.map((machine) => machine.id).sort()).toEqual(
3805 [...next.writer.machineIds, ...next.consumer.machineIds].sort(),
3806 );
3807 const writerAfter = live.find((machine) => machine.id === initial.writer.machineId);
3808 expect(writerAfter?.instance_id).toBe(writerBefore?.instance_id);
3809 expect(writerAfter?.image_ref?.digest).toBe(writerBefore?.image_ref?.digest);
3810 for (const id of next.consumer.machineIds) {
3811 const machine = live.find((machine) => machine.id === id);
3812 expect(machine?.config?.metadata?.["alchemy.phase"]).toBe("active");
3813 expect(machine?.image_ref?.digest).toBe(initial.consumer.imageRef?.digest);
3814 const observed = yield* marker(initial.app.appName, id);
3815 expect(observed.code).toBe(0);
3816 expect(id === held.machineId ? ["two", "three"] : ["three"]).toContain(
3817 observed.marker,
3818 );
3819 }
3820 expect(yield* probeWriter(initial.app.appName, trigger, "GET")).toEqual(before);
3821 }).pipe(
3822 Effect.ensuring(
3823 Effect.sync(() => {
3824 proxy.clear();
3825 proxy.release();
3826 }),
3827 ),
3828 Effect.scoped,
3829 );
3830 }).pipe(Effect.scoped),
3831 { timeout: 900_000 },
3832 );
3833 },
3834 );
3835
3836 describe.sequential(
3837 "readiness oracle",
3838 { tags: ["unit", "provider:fly", "provider:fly:machine", "provider:fly:service", "local"] },
3839 () => {
3840 const names = ["ready", "servicecheck-00-http-80"];
3841 const reports = (mirror = false) =>
3842 [...names, ...(mirror ? ["bg_deployments_compat-00-http-80"] : [])].map((name) => ({
3843 name,
3844 status: "passing",
3845 }));
3846
3847 // Synthetic journals exercise the oracle only; they are not native Fly evidence.
3848 const trace = (idle = true, retry = false, mirror = false) => {
3849 const events: ReadinessEvent[] = [];
3850 const path = (slot: number) => `/v1/apps/oracle/machines/green-${slot}`;
3851 const metadata = (slot: number, phase: string): Record<string, string> => ({
3852 "alchemy.stack": "oracle",
3853 "alchemy.stage": "pure",
3854 "alchemy.id": "Worker",
3855 "alchemy.type": "Fly.Machine",
3856 "alchemy.instance": "resource-instance",
3857 "alchemy.fqn": "Worker",
3858 "alchemy.replica": String(slot),
3859 "alchemy.generation": "generation",
3860 "alchemy.workload": "workload",
3861 "alchemy.image": "nginx@sha256:fixture",
3862 "alchemy.phase": phase,
3863 "alchemy.readiness-role": "run",
3864 "alchemy.readiness-roles": "run,run",
3865 "alchemy.idle-policy-restored": "true",
3866 ...(phase === "active" ? { "alchemy.checked-instance": `instance-${slot}` } : {}),
3867 });
3868 const request = (input: Omit<ReadinessEvent, "stage" | "sequence">) => {
3869 const event: ReadinessEvent = { ...input, stage: "request", sequence: events.length };
3870 events.push(event);
3871 return event;
3872 };
3873 const reply = (input: ReadinessEvent, fields: Partial<ReadinessEvent> = {}) => {
3874 const event: ReadinessEvent = { ...input, stage: "forwarded", status: 200, ...fields };
3875 events.push(event);
3876 return event;
3877 };
3878 const read = (slot: number, phase: string, state: string) =>
3879 reply(request({ method: "GET", path: path(slot), machineId: `green-${slot}` }), {
3880 instanceId: `instance-${slot}`,
3881 digest: "sha256:fixture",
3882 state,
3883 cordoned: false,
3884 phase,
3885 metadata: metadata(slot, phase),
3886 checks: state === "started" ? reports(mirror) : [],
3887 services: [
3888 { protocol: "tcp", port: 80, autostop: "stop", autostart: true, floor: 0 },
3889 ],
3890 });
3891 const stamp = (slot: number, phase: string) => {
3892 const input = {
3893 method: "PATCH",
3894 path: `${path(slot)}/metadata`,
3895 machineId: `green-${slot}`,
3896 phase,
3897 metadata: metadata(slot, phase),
3898 metadataOnly: true,
3899 };
3900 if (retry) reply(request(input), { status: phase === "active" ? 503 : 429 });
3901 reply(request(input));
3902 };
3903 for (const slot of [0, 1]) {
3904 reply(request({ method: "POST", path: path(slot), machineId: `green-${slot}` }));
3905 read(slot, "promoting", "started");
3906 }
3907 for (const slot of [0, 1]) stamp(slot, "validating");
3908 for (const slot of [0, 1])
3909 read(slot, "validating", idle && slot === 0 ? "stopped" : "started");
3910 for (const slot of [1, 0]) stamp(slot, "active");
3911 request({
3912 method: "POST",
3913 path: "/v1/apps/oracle/machines/old/cordon",
3914 machineId: "old",
3915 });
3916 const candidates: Machine[] = [0, 1].map((slot) => ({
3917 id: `green-${slot}`,
3918 instance_id: `instance-${slot}`,
3919 image_ref: { digest: "sha256:fixture" },
3920 state: idle && slot === 0 ? "stopped" : "started",
3921 cordoned: false,
3922 config: {
3923 metadata: metadata(slot, "active"),
3924 services: [
3925 {
3926 protocol: "tcp",
3927 internal_port: 80,
3928 autostop: "stop",
3929 autostart: true,
3930 min_machines_running: 0,
3931 },
3932 ],
3933 },
3934 }));
3935 return { events, candidates };
3936 };
3937
3938 it.effect(
3939 "pure readiness trace accepts rejected resume only with fresh started-instance proof",
3940 () =>
3941 Effect.sync(() => {
3942 const rejectedStart = (status: number) => {
3943 const value = trace(false);
3944 const request: ReadinessEvent = {
3945 sequence: 999,
3946 stage: "request",
3947 method: "POST",
3948 path: "/v1/apps/oracle/machines/green-0/start",
3949 machineId: "green-0",
3950 };
3951 value.events.splice(2, 0, request, { ...request, stage: "forwarded", status });
3952 return value;
3953 };
3954 expect(() => check(rejectedStart(412))).not.toThrow();
3955 expect(() => check(rejectedStart(500))).toThrow();
3956 const missing = rejectedStart(412);
3957 for (const event of missing.events)
3958 if (event.method === "GET" && event.machineId === "green-0")
3959 event.state = "stopped";
3960 expect(() => check(missing)).toThrow();
3961 }),
3962 );
3963
3964 type Trace = ReturnType<typeof trace>;
3965 const check = ({ events, candidates }: Trace, allowIdle = true) =>
3966 assertReadinessCommit(events, ["old"], candidates, [0, 1], names, allowIdle);
3967 const restored = ({ events }: Trace) =>
3968 events.find(
3969 (event) =>
3970 event.stage === "forwarded" &&
3971 event.method === "GET" &&
3972 event.machineId === "green-0" &&
3973 event.phase === "promoting",
3974 )!;
3975 const pending = ({ events }: Trace) =>
3976 events.find(
3977 (event) =>
3978 event.stage === "forwarded" &&
3979 event.method === "GET" &&
3980 event.machineId === "green-0" &&
3981 event.phase === "validating",
3982 )!;
3983 const active = ({ events }: Trace) =>
3984 events.filter(
3985 (event) =>
3986 event.method === "PATCH" && event.machineId === "green-0" && event.phase === "active",
3987 );
3988
3989 it.effect("pure readiness matcher permits only passing corresponding mirrors", () =>
3990 Effect.sync(() => {
3991 expect(readinessChecksPassing(reports(), names)).toBe(true);
3992 expect(readinessChecksPassing(reports(true), names)).toBe(true);
3993 for (const invalid of [
3994 undefined,
3995 [],
3996 reports(true).filter((check) => check.name !== "servicecheck-00-http-80"),
3997 [...reports(), reports()[0]!],
3998 [...reports(true), reports(true).at(-1)!],
3999 [...reports(), { name: "unrelated", status: "passing" }],
4000 [...reports(), { name: "bg_deployments_compat-00-tcp-80", status: "passing" }],
4001 [...reports(), { name: undefined, status: "passing" }],
4002 reports(true).map((check) => ({ ...check, status: "warning" })),
4003 reports(true).map((check) =>
4004 check.name.startsWith("bg_") ? { ...check, status: "critical" } : check,
4005 ),
4006 ])
4007 expect(readinessChecksPassing(invalid, names)).toBe(false);
4008 expect(readinessChecksPassing(reports(), [...names, names[0]!])).toBe(false);
4009 }),
4010 );
4011
4012 it.effect(
4013 "pure readiness traces accept running and restored-idle proof with settled metadata retries",
4014 () =>
4015 Effect.sync(() => {
4016 for (const idle of [false, true]) {
4017 for (const retry of [false, true]) check(trace(idle, retry, true));
4018 }
4019 const suspended = trace();
4020 pending(suspended).state = "suspended";
4021 suspended.candidates[0]!.state = "suspended";
4022 check(suspended);
4023 }),
4024 );
4025
4026 const invalidTraces: [string, (value: Trace) => void][] = [
4027 [
4028 "missing restored passing proof",
4029 (value) => {
4030 restored(value).checks = [];
4031 },
4032 ],
4033 [
4034 "proof from another instance",
4035 (value) => {
4036 restored(value).instanceId = "obsolete";
4037 },
4038 ],
4039 [
4040 "proof from another workload",
4041 (value) => {
4042 restored(value).metadata!["alchemy.workload"] = "obsolete";
4043 },
4044 ],
4045 [
4046 "proof with the preparation service policy",
4047 (value) => {
4048 restored(value).services![0]!.autostop = "off";
4049 },
4050 ],
4051 [
4052 "created representative",
4053 (value) => {
4054 pending(value).state = "created";
4055 },
4056 ],
4057 [
4058 "configuration mutation after restored proof",
4059 (value) => {
4060 const index = value.events.indexOf(restored(value)) + 1;
4061 const event: ReadinessEvent = {
4062 sequence: 10_000,
4063 stage: "request",
4064 method: "POST",
4065 path: "/v1/apps/oracle/machines/green-0",
4066 machineId: "green-0",
4067 };
4068 value.events.splice(index, 0, event, { ...event, stage: "forwarded", status: 200 });
4069 },
4070 ],
4071 [
4072 "GET requested before restoration completed",
4073 (value) => {
4074 const response = restored(value);
4075 const index = value.events.findIndex((event) => event.sequence === response.sequence);
4076 const [request] = value.events.splice(index, 1);
4077 value.events.splice(1, 0, request!);
4078 },
4079 ],
4080 [
4081 "changed pending metadata after proof",
4082 (value) => {
4083 for (const event of value.events.filter(
4084 (event) =>
4085 event.method === "PATCH" &&
4086 event.machineId === "green-0" &&
4087 event.phase === "validating",
4088 )) {
4089 event.metadata = { ...event.metadata, "alchemy.checked-instance": "unproven" };
4090 }
4091 },
4092 ],
4093 [
4094 "conflicting commit retry",
4095 (value) => {
4096 const retry = active(value)
4097 .filter((event) => event.stage === "request")
4098 .at(-1)!;
4099 retry.metadata = { ...retry.metadata, "alchemy.checked-instance": "other" };
4100 },
4101 ],
4102 [
4103 "unsettled rejected attempt",
4104 (value) => {
4105 value.events.splice(value.events.indexOf(active(value)[1]!), 1);
4106 },
4107 ],
4108 [
4109 "retry begins before rejected receipt",
4110 (value) => {
4111 const attempts = active(value);
4112 const index = value.events.indexOf(attempts[2]!);
4113 value.events.splice(index, 1);
4114 value.events.splice(value.events.indexOf(attempts[1]!), 0, attempts[2]!);
4115 },
4116 ],
4117 [
4118 "terminal commit rejection",
4119 (value) => {
4120 active(value).at(-1)!.status = 429;
4121 },
4122 ],
4123 [
4124 "nonretryable metadata rejection",
4125 (value) => {
4126 active(value)[1]!.status = 403;
4127 },
4128 ],
4129 [
4130 "duplicate successful commit",
4131 (value) => {
4132 active(value)[1]!.status = 200;
4133 },
4134 ],
4135 [
4136 "replica zero starts committing before the other commit settles",
4137 (value) => {
4138 const response = value.events.find(
4139 (event) =>
4140 event.machineId === "green-1" &&
4141 event.phase === "active" &&
4142 event.stage === "forwarded" &&
4143 event.status === 200,
4144 )!;
4145 value.events.splice(value.events.indexOf(response), 1);
4146 value.events.splice(value.events.indexOf(active(value)[0]!) + 1, 0, response);
4147 },
4148 ],
4149 [
4150 "retirement before terminal commit receipt",
4151 (value) => {
4152 const retirement = value.events.pop()!;
4153 value.events.splice(value.events.indexOf(active(value).at(-1)!), 0, retirement);
4154 },
4155 ],
4156 [
4157 "commit before complete topology validation",
4158 (value) => {
4159 const first = value.events.findIndex((event) => event.phase === "active");
4160 const commit = value.events.splice(first, 4);
4161 const validation = value.events.findIndex(
4162 (event) =>
4163 event.method === "GET" &&
4164 event.machineId === "green-1" &&
4165 event.stage === "forwarded" &&
4166 event.phase === "validating",
4167 );
4168 value.events.splice(validation - 1, 0, ...commit);
4169 },
4170 ],
4171 ];
4172
4173 for (const [name, corrupt] of invalidTraces) {
4174 it.effect(`pure readiness trace rejects ${name}`, () =>
4175 Effect.sync(() => {
4176 const value = trace(true, true);
4177 corrupt(value);
4178 expect(() => check(value)).toThrow();
4179 }),
4180 );
4181 }
4182
4183 it.effect("pure readiness trace refuses idle completion when autostop is disabled", () =>
4184 Effect.sync(() => {
4185 expect(() => check(trace(), false)).toThrow();
4186 }),
4187 );
4188 },
4189 );
4190
4191 describe.sequential(
4192 "engine state",
4193 {
4194 tags: [
4195 "provider:fly",
4196 "provider:fly:app",
4197 "provider:fly:machine",
4198 "provider:fly:service",
4199 "live",
4200 ],
4201 },
4202 () => {
4203 const file = "test/Fly/BlueGreen.test.ts";
4204
4205 const { test } = Test.make({ providers: Fly.providers() });
4206
4207 test.provider(
4208 "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
4209 (stack) =>
4210 Effect.gen(function* () {
4211 yield* stack.destroy();
4212 const proxy = yield* transportProxy();
4213 const firstActor = yield* engineActor(
4214 stack,
4215 "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
4216 file,
4217 proxy.url,
4218 );
4219 const first = yield* firstActor.deploy(Fly.App("Site")).pipe(Effect.scoped);
4220 const secondActor = yield* engineActor(
4221 stack,
4222 "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
4223 file,
4224 proxy.url,
4225 );
4226 expect(secondActor.state).not.toBe(firstActor.state);
4227 const second = yield* secondActor.deploy(Fly.App("Site")).pipe(Effect.scoped);
4228 expect(second.appName).toBe(first.appName);
4229 expect(second.appId).toBe(first.appId);
4230 expect(
4231 proxy.events.filter(
4232 (event) =>
4233 event.stage === "completed" &&
4234 event.method === "POST" &&
4235 event.path === "/v1/apps" &&
4236 event.status! < 300,
4237 ),
4238 ).toHaveLength(1);
4239 yield* stack.destroy();
4240 yield* assertAppGone(first.appName);
4241 }).pipe(Effect.scoped),
4242 { timeout: 180_000 },
4243 );
4244 },
4245 );
4246 });
4247 describe.sequential("shard 4", () => {
4248 describe.sequential(
4249 "check cadence",
4250 {
4251 tags: [
4252 "provider:fly",
4253 "provider:fly:app",
4254 "provider:fly:machine",
4255 "provider:fly:service",
4256 "live",
4257 ],
4258 },
4259 () => {
4260 const file = "test/Fly/BlueGreen.test.ts";
4261 const { test } = Test.make({ providers: Fly.providers() });
4262
4263 const cadenceCheck = {
4264 ...checks.ready,
4265 path: "/cadence-ready",
4266 interval: "75s",
4267 gracePeriod: "0s",
4268 };
4269
4270 describe.sequential("live long-cadence readiness and service interval limits", () => {
4271 for (const kind of ["named", "service"] as const) {
4272 for (const sufficient of [false, true]) {
4273 const budget = sufficient ? 120_000 : 35_000;
4274
4275 test.provider(
4276 `S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`,
4277 (stack) =>
4278 Effect.gen(function* () {
4279 yield* stack.destroy();
4280 const site = yield* stack.deploy(Fly.App("Site"));
4281 try {
4282 const first = yield* deployWorker(stack, "one");
4283 const proxy = yield* readinessProxy();
4284 const actor = yield* readinessActor(
4285 stack,
4286 `S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`,
4287 file,
4288 proxy,
4289 );
4290 const props: Partial<Omit<Extract<MachineProps, { image: string }>, "app">> =
4291 {
4292 init: {
4293 exec: [
4294 "/bin/sh",
4295 "-c",
4296 "rm -f /usr/share/nginx/html/cadence-ready; (sleep 80; touch /usr/share/nginx/html/cadence-ready) & exec nginx -g 'daemon off;'",
4297 ],
4298 },
4299 checks: kind === "named" ? { ready: cadenceCheck } : {},
4300 services:
4301 kind === "service"
4302 ? [
4303 {
4304 protocol: "tcp",
4305 internalPort: 80,
4306 autostop: "off",
4307 checks: [cadenceCheck],
4308 },
4309 ]
4310 : [],
4311 deploy: { strategy: "bluegreen", healthTimeout: budget },
4312 };
4313 yield* Effect.gen(function* () {
4314 const started = yield* Clock.currentTimeMillis;
4315 const attempt = yield* deployWorker(actor, "two", props).pipe(
4316 Effect.scoped,
4317 Effect.result,
4318 Effect.forkScoped,
4319 );
4320 const checkName = kind === "named" ? "ready" : "servicecheck-00-http-80";
4321 const failed = yield* census(site.appName).pipe(
4322 Effect.map((live) =>
4323 live.find(
4324 (machine) =>
4325 !first.machineIds.includes(machine.id!) &&
4326 machine.state === "started" &&
4327 machine.checks?.some(
4328 (check) =>
4329 check.name === checkName && check.status === "critical",
4330 ),
4331 ),
4332 ),
4333 Effect.repeat({
4334 schedule: Schedule.spaced("1 second"),
4335 times: 120,
4336 until: (machine) => machine !== undefined,
4337 }),
4338 Effect.timeout("150 seconds"),
4339 Effect.raceFirst(
4340 Effect.gen(function* () {
4341 const result = yield* Fiber.join(attempt);
4342 if (Result.isFailure(result))
4343 return yield* Effect.fail(result.failure);
4344 return yield* Effect.fail(
4345 new Error(
4346 "Candidate committed before a failing long-cadence report was observed",
4347 ),
4348 );
4349 }),
4350 ),
4351 );
4352 expect(failed).toBeDefined();
4353 if (!failed)
4354 return yield* Effect.fail(
4355 new Error("Fly never emitted the initial failing cadence report"),
4356 );
4357 const nativeCheck =
4358 kind === "named"
4359 ? failed.config?.checks?.ready
4360 : failed.config?.services?.[0]?.checks?.[0];
4361 expect(
4362 kind === "named" ? ["75s", "1m15s"] : ["60s", "1m0s", "1m"],
4363 ).toContain(nativeCheck?.interval);
4364 expect(["0s", "0", undefined]).toContain(nativeCheck?.grace_period);
4365 expect(failed.cordoned).toBe(true);
4366 const failureReport = failed.checks!.find(
4367 (check) => check.name === checkName,
4368 )!;
4369 expect(failureReport.updated_at).toBeDefined();
4370 if (sufficient) {
4371 const passing = yield* machines
4372 .getMachine({ app_name: site.appName, machine_id: failed.id! })
4373 .pipe(
4374 Effect.repeat({
4375 schedule: Schedule.spaced("1 second"),
4376 times: 120,
4377 until: (machine) =>
4378 machine.checks?.some(
4379 (check) =>
4380 check.name === checkName && check.status === "passing",
4381 ) === true,
4382 }),
4383 Effect.timeout("120 seconds"),
4384 );
4385 const report = passing.checks?.find((check) => check.name === checkName);
4386 expect(report?.status).toBe("passing");
4387 expect(report?.updated_at).toBeDefined();
4388 expect(failed.instance_id).toBeDefined();
4389 expect(passing.instance_id).toBe(failed.instance_id);
4390 const reportSpacing = yield* Effect.sync(
4391 () =>
4392 Date.parse(report!.updated_at!) -
4393 Date.parse(failureReport.updated_at!),
4394 );
4395 expect(reportSpacing).toBeGreaterThan(60_000);
4396 yield* Effect.logInfo(
4397 "Native cadence reports across autonomous readiness transition",
4398 {
4399 reportSpacingMs: reportSpacing,
4400 failingReport: failureReport.updated_at,
4401 passingReport: report!.updated_at,
4402 },
4403 );
4404 const result = yield* Fiber.join(attempt).pipe(
4405 Effect.timeout("600 seconds"),
4406 );
4407 expect(Result.isSuccess(result)).toBe(true);
4408 if (Result.isFailure(result)) return yield* Effect.fail(result.failure);
4409 expect(result.success.machineIds).toEqual([failed.id]);
4410 const committed = yield* assertCommitted(
4411 site.appName,
4412 result.success.machineIds,
4413 );
4414 expect(committed[0]!.instance_id).toBe(failed.instance_id);
4415 assertReadinessCommit(
4416 proxy.readiness,
4417 first.machineIds,
4418 committed,
4419 [0],
4420 [checkName],
4421 false,
4422 );
4423 } else {
4424 const result = yield* Fiber.join(attempt).pipe(
4425 Effect.timeout("180 seconds"),
4426 );
4427 expect(Result.isFailure(result)).toBe(true);
4428 if (Result.isFailure(result))
4429 expect(result.failure._tag).toBe("Fly.ReplicaChecksNotPassing");
4430 expect((yield* Clock.currentTimeMillis) - started).toBeLessThan(180_000);
4431 const live = yield* census(site.appName);
4432 expect(live.map((machine) => machine.id)).toEqual(first.machineIds);
4433 expect(live[0]!.state).toBe("started");
4434 expect(live[0]!.cordoned).toBe(false);
4435 expect(
4436 proxy.events.some(
4437 (event) =>
4438 event.stage === "request" &&
4439 first.machineIds.includes(event.machineId!) &&
4440 (event.path.endsWith("/stop") ||
4441 event.path.endsWith("/cordon") ||
4442 event.path.endsWith("/suspend") ||
4443 (event.path.endsWith("/metadata") &&
4444 event.phase === "retiring") ||
4445 (event.method === "DELETE" &&
4446 /\/machines\/[^/]+$/.test(event.path))),
4447 ),
4448 ).toBe(false);
4449 expect(
4450 proxy.events.some(
4451 (event) =>
4452 event.machineId === failed.id && event.path.endsWith("/uncordon"),
4453 ),
4454 ).toBe(false);
4455 expect(
4456 proxy.readiness.some(
4457 (event) =>
4458 event.stage === "request" &&
4459 event.machineId === failed.id &&
4460 event.path.endsWith("/metadata") &&
4461 event.phase === "active",
4462 ),
4463 ).toBe(false);
4464 }
4465 yield* Effect.logInfo("Live long-cadence evidence", {
4466 kind,
4467 interval: nativeCheck?.interval,
4468 grace: nativeCheck?.grace_period,
4469 budget,
4470 firstFailure: failureReport.updated_at,
4471 elapsedMs: (yield* Clock.currentTimeMillis) - started,
4472 });
4473 }).pipe(Effect.scoped);
4474 } finally {
4475 yield* stack.destroy();
4476 yield* assertAppGone(site.appName);
4477 }
4478 }).pipe(Effect.scoped),
4479 { timeout: 1_200_000 },
4480 );
4481 }
4482 }
4483 });
4484 },
4485 );
4486
4487 describe.sequential(
4488 "multi-container",
4489 { tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"] },
4490 () => {
4491 const { test } = Test.make({ providers: Fly.providers() });
4492 const firstImage =
4493 "docker-hub-mirror.fly.io/library/nginx@sha256:7396be67b6f53012a5cf955fa9040619294c25ccacf11e22af5de1b572fc756e";
4494 const nextImage =
4495 "docker-hub-mirror.fly.io/library/nginx@sha256:ef8676b33d681f272ba429b27658bdd7e640963279714c96bddf1dc76307f7b6";
4496
4497 interface GroupOptions {
4498 count?: number;
4499 /** Reverse declaration order and add empty optional fields. */
4500 reordered?: boolean;
4501 /** Add a third container to the group. */
4502 extra?: boolean;
4503 badHealth?: boolean;
4504 strategy?: "rolling" | "bluegreen";
4505 }
4506
4507 const containers = (
4508 sidecarImage: string,
4509 { reordered = false, extra = false }: GroupOptions,
4510 ): MachineContainer[] => {
4511 const group: MachineContainer[] = [
4512 {
4513 name: "web",
4514 image: firstImage,
4515 healthChecks: [
4516 { name: "web-tcp", kind: "readiness", tcp: { port: 80 }, interval: 5, timeout: 2 },
4517 ],
4518 },
4519 {
4520 name: "sidecar",
4521 image: sidecarImage,
4522 cmd: [
4523 "sh",
4524 "-c",
4525 "sed -i 's/80/8080/g' /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'",
4526 ],
4527 dependsOn: [{ name: "web", condition: "healthy" }],
4528 healthChecks: [
4529 {
4530 name: "sidecar-tcp",
4531 kind: "readiness",
4532 tcp: { port: 8080 },
4533 interval: 5,
4534 timeout: 2,
4535 },
4536 ],
4537 },
4538 ...(extra
4539 ? [{ name: "extra", image: firstImage, cmd: ["sh", "-c", "sleep infinity"] }]
4540 : []),
4541 ];
4542 return reordered
4543 ? group.reverse().map((container) => ({ ...container, env: {} }))
4544 : group;
4545 };
4546
4547 const machineChecks = (badHealth: boolean) => ({
4548 web: { type: "tcp" as const, port: 80, interval: "2s", timeout: "1s" },
4549 sidecar: {
4550 type: "tcp" as const,
4551 port: badHealth ? 9999 : 8080,
4552 interval: "2s",
4553 timeout: "1s",
4554 },
4555 });
4556
4557 const deployGroup = (
4558 stack: ScratchStack,
4559 sidecarImage: string,
4560 options: GroupOptions = {},
4561 ) =>
4562 stack.deploy(
4563 Effect.gen(function* () {
4564 const app = yield* Fly.App("MultiContainerBlueGreenSite");
4565 return yield* Fly.Machine("Group", {
4566 app,
4567 region: "fra",
4568 count: options.count ?? 1,
4569 guest: { cpus: 1, memoryMb: 256 },
4570 containers: containers(sidecarImage, options),
4571 checks: machineChecks(options.badHealth ?? false),
4572 deploy: {
4573 strategy: options.strategy ?? "bluegreen",
4574 healthTimeout: options.badHealth ? "15 seconds" : "30 seconds",
4575 },
4576 shutdown: { signal: "SIGTERM", timeout: "1 second" },
4577 });
4578 }),
4579 );
4580
4581 const images = (machine: Machine) =>
4582 (machine.config?.containers ?? [])
4583 .map(({ name, image }) => ({ name, image }))
4584 .sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
4585
4586 const appGone = (appName: string) =>
4587 machines.getApp({ app_name: appName }).pipe(
4588 Effect.as(false),
4589 Effect.catchTag("NotFound", () => Effect.succeed(true)),
4590 );
4591
4592 const machineGone = (appName: string, machineId: string) =>
4593 machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
4594 Effect.map((machine) => machine.state === "destroyed"),
4595 Effect.catchTag("NotFound", () => Effect.succeed(true)),
4596 Effect.repeat({
4597 schedule: Schedule.spaced("2 seconds"),
4598 times: 10,
4599 until: (gone) => gone,
4600 }),
4601 );
4602
4603 test.provider(
4604 "checks two named-container replicas and ignores declaration order and empty fields",
4605 (stack) =>
4606 Effect.gen(function* () {
4607 yield* stack.destroy();
4608 const first = yield* deployGroup(stack, firstImage, { count: 2 });
4609 expect(first.machineIds).toHaveLength(2);
4610 const observed = yield* Effect.forEach(first.machineIds, (id) =>
4611 machines.getMachine({ app_name: first.appName, machine_id: id }),
4612 );
4613 for (const machine of observed) {
4614 expect(images(machine)).toEqual([
4615 { name: "sidecar", image: firstImage },
4616 { name: "web", image: firstImage },
4617 ]);
4618 expect(machine.cordoned).toBe(false);
4619 expect(machine.config?.metadata?.[keys.phase]).toBe("active");
4620 expect(machine.config?.metadata?.[keys.protocol]).toBe("2");
4621 const ready = yield* waitHealthy(first.appName, machine, 30_000);
4622 for (const check of ["web", "sidecar"])
4623 expect(
4624 ready.checks?.some(
4625 ({ name, status }) => name === check && status === "passing",
4626 ),
4627 ).toBe(true);
4628 }
4629 const reordered = yield* deployGroup(stack, firstImage, {
4630 count: 2,
4631 reordered: true,
4632 });
4633 expect([...reordered.machineIds].sort()).toEqual([...first.machineIds].sort());
4634 for (const machine of observed) {
4635 const after = yield* machines.getMachine({
4636 app_name: first.appName,
4637 machine_id: machine.id!,
4638 });
4639 expect(after.instance_id).toBe(machine.instance_id);
4640 }
4641 yield* stack.destroy();
4642 expect(yield* appGone(first.appName)).toBe(true);
4643 }),
4644 { timeout: 180_000 },
4645 );
4646
4647 test.provider(
4648 "named group transitions from rolling to blue/green and back",
4649 (stack) =>
4650 Effect.gen(function* () {
4651 yield* stack.destroy();
4652 const rolling = yield* deployGroup(stack, firstImage, { strategy: "rolling" });
4653 const promoted = yield* deployGroup(stack, firstImage);
4654 expect(promoted.machineId).not.toBe(rolling.machineId);
4655 const promotedMachine = yield* machines.getMachine({
4656 app_name: promoted.appName,
4657 machine_id: promoted.machineId,
4658 });
4659 expect(promotedMachine.config?.metadata?.[keys.protocol]).toBe("2");
4660 expect(images(promotedMachine)).toEqual([
4661 { name: "sidecar", image: firstImage },
4662 { name: "web", image: firstImage },
4663 ]);
4664 const optedOut = yield* deployGroup(stack, firstImage, { strategy: "rolling" });
4665 expect(optedOut.machineId).toBe(promoted.machineId);
4666 const live = yield* machines.getMachine({
4667 app_name: optedOut.appName,
4668 machine_id: optedOut.machineId,
4669 });
4670 expect(live.config?.metadata?.[keys.protocol]).toBeUndefined();
4671 expect(images(live)).toEqual([
4672 { name: "sidecar", image: firstImage },
4673 { name: "web", image: firstImage },
4674 ]);
4675 yield* stack.destroy();
4676 expect(yield* appGone(rolling.appName)).toBe(true);
4677 }),
4678 { timeout: 180_000 },
4679 );
4680
4681 test.provider(
4682 "a single-image blue/green generation is replaced by a named group",
4683 (stack) =>
4684 Effect.gen(function* () {
4685 yield* stack.destroy();
4686 const single = yield* stack.deploy(
4687 Effect.gen(function* () {
4688 const app = yield* Fly.App("MultiContainerBlueGreenSite");
4689 return yield* Fly.Machine("Group", {
4690 app,
4691 region: "fra",
4692 guest: { cpus: 1, memoryMb: 256 },
4693 image: firstImage,
4694 checks: { web: machineChecks(false).web },
4695 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
4696 shutdown: { signal: "SIGTERM", timeout: "1 second" },
4697 });
4698 }),
4699 );
4700 const before = yield* machines.getMachine({
4701 app_name: single.appName,
4702 machine_id: single.machineId,
4703 });
4704 expect(before.config?.metadata?.[keys.protocol]).toBe("1");
4705 const group = yield* deployGroup(stack, firstImage);
4706 expect(group.machineId).not.toBe(single.machineId);
4707 const after = yield* machines.getMachine({
4708 app_name: group.appName,
4709 machine_id: group.machineId,
4710 });
4711 expect(after.config?.metadata?.[keys.protocol]).toBe("2");
4712 expect(after.config?.metadata?.[keys.phase]).toBe("active");
4713 expect(images(after)).toEqual([
4714 { name: "sidecar", image: firstImage },
4715 { name: "web", image: firstImage },
4716 ]);
4717 expect(yield* machineGone(single.appName, single.machineId)).toBe(true);
4718 yield* stack.destroy();
4719 expect(yield* appGone(single.appName)).toBe(true);
4720 }),
4721 { timeout: 180_000 },
4722 );
4723
4724 test.provider(
4725 "changing one image or adding a container replaces the whole group",
4726 (stack) =>
4727 Effect.gen(function* () {
4728 yield* stack.destroy();
4729 const first = yield* deployGroup(stack, firstImage);
4730 const updated = yield* deployGroup(stack, nextImage, { reordered: true });
4731 expect(updated.machineId).not.toBe(first.machineId);
4732 const replacement = yield* machines.getMachine({
4733 app_name: updated.appName,
4734 machine_id: updated.machineId,
4735 });
4736 expect(images(replacement)).toEqual([
4737 { name: "sidecar", image: nextImage },
4738 { name: "web", image: firstImage },
4739 ]);
4740 expect(replacement.cordoned).toBe(false);
4741 expect(replacement.config?.metadata?.[keys.phase]).toBe("active");
4742 expect(yield* machineGone(first.appName, first.machineId)).toBe(true);
4743
4744 const extended = yield* deployGroup(stack, nextImage, { extra: true });
4745 expect(extended.machineId).not.toBe(updated.machineId);
4746 expect(
4747 images(
4748 yield* machines.getMachine({
4749 app_name: extended.appName,
4750 machine_id: extended.machineId,
4751 }),
4752 ),
4753 ).toEqual([
4754 { name: "extra", image: firstImage },
4755 { name: "sidecar", image: nextImage },
4756 { name: "web", image: firstImage },
4757 ]);
4758 expect(yield* machineGone(updated.appName, updated.machineId)).toBe(true);
4759 yield* stack.destroy();
4760 expect(yield* appGone(first.appName)).toBe(true);
4761 }),
4762 { timeout: 240_000 },
4763 );
4764
4765 test.provider(
4766 "bad secondary readiness preserves the serving predecessor",
4767 (stack) =>
4768 Effect.gen(function* () {
4769 yield* stack.destroy();
4770 const first = yield* deployGroup(stack, firstImage);
4771 const failed = yield* deployGroup(stack, nextImage, { badHealth: true }).pipe(
4772 Effect.result,
4773 );
4774 expect(Result.isFailure(failed)).toBe(true);
4775 const predecessor = yield* machines.getMachine({
4776 app_name: first.appName,
4777 machine_id: first.machineId,
4778 });
4779 expect(predecessor.state).toBe("started");
4780 expect(predecessor.cordoned).toBe(false);
4781 expect(predecessor.config?.metadata?.[keys.phase]).toBe("active");
4782 expect(images(predecessor)).toEqual([
4783 { name: "sidecar", image: firstImage },
4784 { name: "web", image: firstImage },
4785 ]);
4786 yield* stack.destroy();
4787 expect(yield* appGone(first.appName)).toBe(true);
4788 }),
4789 { timeout: 180_000 },
4790 );
4791
4792 test.provider(
4793 "tampered deployment metadata preserves the serving group",
4794 (stack) =>
4795 Effect.gen(function* () {
4796 yield* stack.destroy();
4797 const first = yield* deployGroup(stack, firstImage);
4798 const target = { app_name: first.appName, machine_id: first.machineId };
4799 const original = yield* machines.getMachine(target);
4800 const tamperings: Array<[string, Record<string, string>]> = [
4801 ["unknown protocol", { [keys.protocol]: "future" }],
4802 [
4803 "incomplete image set",
4804 {
4805 [keys.containerImageSet]: JSON.stringify([{ name: "web", image: firstImage }]),
4806 },
4807 ],
4808 ];
4809 for (const [label, overrides] of tamperings) {
4810 // Rewrite Alchemy's ownership metadata directly through the Fly API.
4811 const current = yield* machines.getMachine(target);
4812 const tampered = yield* machines.updateMachine({
4813 ...target,
4814 config: {
4815 ...current.config,
4816 metadata: { ...original.config?.metadata, ...overrides },
4817 },
4818 });
4819 yield* machines.waitMachine({
4820 ...target,
4821 state: "started",
4822 instance_id: tampered.instance_id,
4823 timeout: 30,
4824 });
4825 const failed = yield* deployGroup(stack, nextImage).pipe(Effect.flip);
4826 expect(failed, label).toBeInstanceOf(DeploymentRecoveryAmbiguous);
4827 const preserved = yield* machines.getMachine(target);
4828 expect(preserved.instance_id, label).toBe(tampered.instance_id);
4829 expect(preserved.state, label).toBe("started");
4830 expect(images(preserved), label).toEqual([
4831 { name: "sidecar", image: firstImage },
4832 { name: "web", image: firstImage },
4833 ]);
4834 const inventory = (yield* machines.listMachines({
4835 app_name: first.appName,
4836 })).filter((machine) => machine.state !== "destroyed");
4837 expect(
4838 inventory.map(({ id }) => id),
4839 label,
4840 ).toEqual([first.machineId]);
4841 }
4842 const restored = yield* machines.updateMachine({
4843 ...target,
4844 config: {
4845 ...(yield* machines.getMachine(target)).config,
4846 metadata: original.config?.metadata,
4847 },
4848 });
4849 yield* machines.waitMachine({
4850 ...target,
4851 state: "started",
4852 instance_id: restored.instance_id,
4853 timeout: 30,
4854 });
4855 yield* stack.destroy();
4856 expect(yield* appGone(first.appName)).toBe(true);
4857 }),
4858 { timeout: 180_000 },
4859 );
4860 },
4861 );
4862
4863 describe.sequential(
4864 "fiber interruption",
4865 {
4866 tags: [
4867 "provider:fly",
4868 "provider:fly:app",
4869 "provider:fly:machine",
4870 "provider:fly:service",
4871 "live",
4872 ],
4873 },
4874 () => {
4875 const file = "test/Fly/BlueGreen.test.ts";
4876 const { test } = Test.make({ providers: Fly.providers() });
4877
4878 describe.sequential("in-process engine interruption", () => {
4879 for (const phase of ["create", "promotion", "retirement"] as const) {
4880 test.provider(
4881 `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
4882 (stack) =>
4883 Effect.gen(function* () {
4884 yield* stack.destroy();
4885 const initial = yield* deployWorker(stack, "one");
4886 const proxy = yield* transportProxy();
4887 const match = (event: TransportEvent) =>
4888 phase === "create"
4889 ? event.method === "POST" && event.path.endsWith("/machines")
4890 : phase === "promotion"
4891 ? event.path.endsWith("/uncordon")
4892 : event.method === "DELETE" &&
4893 /\/machines\/[^/]+$/.test(event.path) &&
4894 event.machineId === initial.machineId;
4895 yield* Effect.sync(() =>
4896 proxy.arm({ match, action: "hold-response", remaining: 1 }),
4897 );
4898 yield* Effect.gen(function* () {
4899 const actor = yield* engineActor(
4900 stack,
4901 `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
4902 file,
4903 proxy.url,
4904 );
4905 const interrupted = yield* deployWorker(actor, "two").pipe(
4906 Effect.scoped,
4907 Effect.forkScoped,
4908 );
4909 const barrier = yield* proxy.wait(
4910 (event) =>
4911 event.stage === "held" &&
4912 event.status! >= 200 &&
4913 event.status! < 300 &&
4914 match(event),
4915 );
4916 expect(barrier.machineId).toBeDefined();
4917 const interruption = yield* Fiber.interrupt(interrupted).pipe(
4918 Effect.forkScoped,
4919 );
4920 yield* Effect.yieldNow;
4921 // Let accepted uninterruptible work and finalizers settle; this is not process-kill evidence.
4922 yield* Effect.sync(() => {
4923 proxy.clear();
4924 proxy.release();
4925 });
4926 yield* Fiber.join(interruption).pipe(Effect.timeout("90 seconds"));
4927 const exit = yield* Fiber.await(interrupted);
4928 expect(Exit.hasInterrupts(exit)).toBe(true);
4929 const surviving = yield* census(initial.appName);
4930 const green = surviving.filter((machine) => machine.id !== initial.machineId);
4931 if (phase === "create") {
4932 expect(surviving.some((machine) => machine.id === initial.machineId)).toBe(
4933 true,
4934 );
4935 expect(green.length).toBeLessThanOrEqual(1);
4936 } else {
4937 expect(green).toHaveLength(1);
4938 expect(green[0]!.cordoned).toBe(false);
4939 }
4940 const resumed = yield* engineActor(
4941 stack,
4942 `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
4943 file,
4944 );
4945 expect(resumed.state).not.toBe(actor.state);
4946 const recovered = yield* deployWorker(resumed, "two").pipe(Effect.scoped);
4947 if (green.length)
4948 expect(recovered.machineIds).toEqual(green.map((machine) => machine.id));
4949 yield* assertCommitted(initial.appName, recovered.machineIds);
4950 }).pipe(
4951 Effect.ensuring(
4952 Effect.sync(() => {
4953 proxy.clear();
4954 proxy.release();
4955 }),
4956 ),
4957 Effect.scoped,
4958 );
4959 yield* stack.destroy();
4960 yield* assertAppGone(initial.appName);
4961 }).pipe(Effect.scoped),
4962 { timeout: 300_000 },
4963 );
4964 }
4965 });
4966 },
4967 );
4968
4969 describe.sequential(
4970 "multi-container public traffic",
4971 { tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"] },
4972 () => {
4973 const { test } = Test.make({ providers: Fly.providers() });
4974 const image =
4975 "docker-hub-mirror.fly.io/library/node@sha256:b6f26b36c8ff49624cfdac716b8ea1138d606df02586a77d364bb5536a634f85";
4976
4977 const scenario = (stack: ScratchStack) =>
4978 Effect.gen(function* () {
4979 const fs = yield* FileSystem.FileSystem;
4980 const path = yield* Path.Path;
4981 const script = yield* fs.readFileString(
4982 yield* path.fromFileUrl(
4983 new URL("./fixtures/multi-container-http.ts", import.meta.url),
4984 ),
4985 );
4986 return (version: string, badHealth = false) =>
4987 stack.deploy(
4988 Effect.gen(function* () {
4989 const app = yield* Fly.App("Traffic");
4990 yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
4991 return yield* Fly.Machine("Group", {
4992 app,
4993 region: "fra",
4994 guest: { cpus: 1, memoryMb: 256 },
4995 containers: [
4996 {
4997 name: "web",
4998 image,
4999 cmd: ["node", "--input-type=module-typescript", "-e", script],
5000 env: { PORT: "3000", CONTAINER_NAME: "web", VERSION: version },
Showing first 5,000 of 9,884 lines. View raw