The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { randomBytes, createHash } from "node:crypto"; |
| 2 | import * as Docker from "@alchemy.run/docker"; |
| 3 | import * as Fly from "@alchemy.run/fly"; |
| 4 | import { AppDeletionAmbiguous } from "@alchemy.run/fly/App"; |
| 5 | import { readinessRoles, DeploymentRecoveryAmbiguous } from "@alchemy.run/fly/bluegreen"; |
| 6 | import { makeMachineLeases } from "@alchemy.run/fly/leases"; |
| 7 | import { type MachineContainer, type MachineProps } from "@alchemy.run/fly/Machine"; |
| 8 | import { alchemyMetadataKeys as keys } from "@alchemy.run/fly/Metadata"; |
| 9 | import { |
| 10 | waitHealthy, |
| 11 | autostopMode, |
| 12 | checksPassing, |
| 13 | ensureStarted, |
| 14 | observeReplicaSet, |
| 15 | predecessorShutdown, |
| 16 | deleteReplicaSet, |
| 17 | ReplicaNotCreated, |
| 18 | ReplicaRetirementIncomplete, |
| 19 | retireMachines, |
| 20 | retireMachine, |
| 21 | } from "@alchemy.run/fly/replicas"; |
| 22 | import { ReplicaChecksNotPassing } from "@alchemy.run/fly/replicas"; |
| 23 | import { expect, assert, it, describe } from "@alchemy.run/test"; |
| 24 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 25 | import type { ScratchStack } from "@alchemy.run/test/Alchemy"; |
| 26 | import * as TestCore from "@alchemy.run/test/Core"; |
| 27 | import { scratchStack, withProviders } from "@alchemy.run/test/Core"; |
| 28 | import { |
| 29 | GatewayTimeout, |
| 30 | HTTP_STATUS_MAP, |
| 31 | RETRYABLE_HTTP_STATUSES, |
| 32 | } from "@distilled.cloud/core/errors"; |
| 33 | import { Credentials } from "@distilled.cloud/fly-io/Credentials"; |
| 34 | import { GatewayTimeout as GatewayTimeoutErrors } from "@distilled.cloud/fly-io/Errors"; |
| 35 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 36 | import { type Machine } from "@distilled.cloud/fly-io/machines"; |
| 37 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 38 | import * as Alchemy from "alchemy"; |
| 39 | import { DestroyError } from "alchemy/Apply"; |
| 40 | import { localState, makeLocalState } from "alchemy/State/LocalState"; |
| 41 | import * as Cause from "effect/Cause"; |
| 42 | import * as Clock from "effect/Clock"; |
| 43 | import * as ConfigProvider from "effect/ConfigProvider"; |
| 44 | import * as Data from "effect/Data"; |
| 45 | import * as Deferred from "effect/Deferred"; |
| 46 | import * as Effect from "effect/Effect"; |
| 47 | import * as Exit from "effect/Exit"; |
| 48 | import * as Fiber from "effect/Fiber"; |
| 49 | import * as FileSystem from "effect/FileSystem"; |
| 50 | import * as FetchHttpClient from "effect/http/FetchHttpClient"; |
| 51 | import * as HttpClient from "effect/http/HttpClient"; |
| 52 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 53 | import * as Path from "effect/Path"; |
| 54 | import * as Redacted from "effect/Redacted"; |
| 55 | import * as Ref from "effect/Ref"; |
| 56 | import * as Result from "effect/Result"; |
| 57 | import * as Schedule from "effect/Schedule"; |
| 58 | import * as Schema from "effect/Schema"; |
| 59 | import * as Stream from "effect/Stream"; |
| 60 | import { engineActor } from "./fixtures/actors.ts"; |
| 61 | import { dropCompletedCreate } from "./fixtures/bluegreen-create-proxy.ts"; |
| 62 | import { |
| 63 | Site, |
| 64 | Token, |
| 65 | TRIGGER_SECRET, |
| 66 | Writer, |
| 67 | writerLayer, |
| 68 | } from "./fixtures/bluegreen-runtime-secrets/writer.ts"; |
| 69 | import { |
| 70 | BoundSecrets, |
| 71 | CacheOne, |
| 72 | CacheTwo, |
| 73 | Site as SiteBluegreenSecrets, |
| 74 | } from "./fixtures/bluegreen-secrets.ts"; |
| 75 | import { |
| 76 | assertOrder, |
| 77 | assertReplacement, |
| 78 | assertStopped, |
| 79 | makeScenario, |
| 80 | requireValue, |
| 81 | } from "./fixtures/bluegreen-worker-test.ts"; |
| 82 | import { |
| 83 | assertAppGone, |
| 84 | census, |
| 85 | checks, |
| 86 | deployWorker, |
| 87 | assertCommitted, |
| 88 | } from "./fixtures/bluegreen.ts"; |
| 89 | import { sanitizeExecFailure } from "./fixtures/exec-lease.ts"; |
| 90 | import { makeReadinessControl, repairReadiness } from "./fixtures/http-readiness-control.ts"; |
| 91 | import { |
| 92 | assertReadinessCommit, |
| 93 | readinessActor, |
| 94 | readinessChecksPassing, |
| 95 | readinessProxy, |
| 96 | retires, |
| 97 | type ReadinessEvent, |
| 98 | } from "./fixtures/idle-cadence-readiness.ts"; |
| 99 | import { |
| 100 | observeStops, |
| 101 | type StopRequest, |
| 102 | writeLegacyProtocol, |
| 103 | } from "./fixtures/legacy-protocol-writer.ts"; |
| 104 | import MountedBlueGreen from "./fixtures/mounted-bluegreen.ts"; |
| 105 | import { |
| 106 | assertBarrierInventory, |
| 107 | assertClean, |
| 108 | assertConverged, |
| 109 | assertSingleRunner, |
| 110 | census as censusProcessDeath, |
| 111 | deploy, |
| 112 | evidencePaths, |
| 113 | heldLeases, |
| 114 | identity, |
| 115 | machine, |
| 116 | matchesBarrier, |
| 117 | nowSeconds, |
| 118 | observeLeaseExpiry, |
| 119 | persistedRow, |
| 120 | phases, |
| 121 | processDeathFile, |
| 122 | readWitness, |
| 123 | writeEvidence, |
| 124 | type Phase, |
| 125 | type Witness, |
| 126 | } from "./fixtures/process-death.ts"; |
| 127 | import { |
| 128 | appName, |
| 129 | candidateId, |
| 130 | MachineServiceAutostop, |
| 131 | metadata, |
| 132 | protocolClient, |
| 133 | reconcile, |
| 134 | reply, |
| 135 | withControlledClient, |
| 136 | } from "./fixtures/protocol-branches.ts"; |
| 137 | import { |
| 138 | Finalized, |
| 139 | RunnerInterrupted, |
| 140 | Witness as WitnessSignalOverlap, |
| 141 | assertBoundary, |
| 142 | assertConverged as assertConvergedSignalOverlap, |
| 143 | assertInventory, |
| 144 | assertReleased, |
| 145 | assertSingleRunner as assertSingleRunnerSignalOverlap, |
| 146 | boundary, |
| 147 | cases, |
| 148 | firstReturnedUncordon, |
| 149 | matches, |
| 150 | observeExpiry, |
| 151 | observeLeases, |
| 152 | pathsFor, |
| 153 | readEvidence, |
| 154 | signalOverlapFile, |
| 155 | successful, |
| 156 | type SignalCase, |
| 157 | } from "./fixtures/signal-overlap.ts"; |
| 158 | import { delayedResourceWrite, ResourceRow } from "./fixtures/state-persistence.ts"; |
| 159 | import { transportProxy, throughProxy, type TransportEvent } from "./fixtures/transport.ts"; |
| 160 | |
| 161 | // Blocks are independent (each case deploys its own stack). Fully parallel |
| 162 | // overloads Fly's API (slow App deletes, missed timing windows), so they run |
| 163 | // in six concurrent shards balanced by measured duration (~35 min each); |
| 164 | // blocks run one at a time within a shard, and process-wide cases keep |
| 165 | // `exclusive: true`. |
| 166 | /** URL param values may be `Redacted` (sent unwrapped); compare their plain text. */ |
| 167 | const unredact = (value: string | Redacted.Redacted | undefined) => |
| 168 | Redacted.isRedacted(value) ? String(Redacted.value(value)) : value; |
| 169 | |
| 170 | const decodeAutostop = Schema.decodeUnknownSync(MachineServiceAutostop); |
| 171 | |
| 172 | describe.concurrent("BlueGreen", () => { |
| 173 | describe.sequential("shard 1", () => { |
| 174 | describe.sequential( |
| 175 | "idle topology", |
| 176 | { |
| 177 | tags: [ |
| 178 | "provider:fly", |
| 179 | "provider:fly:app", |
| 180 | "provider:fly:machine", |
| 181 | "provider:fly:service", |
| 182 | "live", |
| 183 | ], |
| 184 | }, |
| 185 | () => { |
| 186 | const file = "test/Fly/BlueGreen.test.ts"; |
| 187 | const { test } = Test.make({ providers: Fly.providers() }); |
| 188 | type IdleMode = "stop" | "suspend"; |
| 189 | |
| 190 | const topology = ( |
| 191 | mode: IdleMode, |
| 192 | count: number, |
| 193 | floor = 0, |
| 194 | mixed = false, |
| 195 | ): Partial<Omit<Extract<MachineProps, { image: string }>, "app">> => ({ |
| 196 | count, |
| 197 | deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" }, |
| 198 | services: [ |
| 199 | { |
| 200 | protocol: "tcp", |
| 201 | internalPort: 80, |
| 202 | autostop: mode, |
| 203 | autostart: true, |
| 204 | minMachinesRunning: floor, |
| 205 | checks: [checks.ready], |
| 206 | }, |
| 207 | ...(mixed ? [{ protocol: "tcp", internalPort: 81, autostop: "off" as const }] : []), |
| 208 | ], |
| 209 | }); |
| 210 | |
| 211 | const waitState = (appName: string, machineId: string, state: string) => |
| 212 | machines.getMachine({ app_name: appName, machine_id: machineId }).pipe( |
| 213 | Effect.repeat({ |
| 214 | schedule: Schedule.spaced("2 seconds"), |
| 215 | times: 30, |
| 216 | until: (machine) => machine.state === state, |
| 217 | }), |
| 218 | Effect.tap((machine) => Effect.sync(() => expect(machine.state).toBe(state))), |
| 219 | Effect.timeout("90 seconds"), |
| 220 | ); |
| 221 | |
| 222 | // Native stop/suspend is setup only; automatic return-to-idle has its own test. |
| 223 | const idleAll = (appName: string, ids: string[], mode: IdleMode) => |
| 224 | Effect.forEach(ids, (id) => |
| 225 | Effect.gen(function* () { |
| 226 | const target = { app_name: appName, machine_id: id }; |
| 227 | const current = yield* machines.getMachine(target); |
| 228 | if (current.state !== "started") { |
| 229 | yield* machines.startMachine(target); |
| 230 | yield* waitState(appName, id, "started"); |
| 231 | } |
| 232 | if (mode === "stop") { |
| 233 | yield* machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" }); |
| 234 | } else { |
| 235 | yield* machines.suspendMachine(target); |
| 236 | } |
| 237 | yield* waitState(appName, id, mode === "stop" ? "stopped" : "suspended"); |
| 238 | }), |
| 239 | ); |
| 240 | |
| 241 | // Expected slots are literal scenario inputs, never computed by readinessRoles. |
| 242 | const assertSlots = (live: machines.Machine[], runningSlots: readonly number[]) => { |
| 243 | const ordered = [...live].sort( |
| 244 | (left, right) => |
| 245 | Number(left.config?.metadata?.["alchemy.replica"]) - |
| 246 | Number(right.config?.metadata?.["alchemy.replica"]), |
| 247 | ); |
| 248 | expect(ordered.map((machine) => machine.config?.metadata?.["alchemy.replica"])).toEqual( |
| 249 | Array.from({ length: live.length }, (_, index) => String(index)), |
| 250 | ); |
| 251 | expect( |
| 252 | ordered |
| 253 | .filter((machine) => machine.config?.metadata?.["alchemy.readiness-role"] === "run") |
| 254 | .map((machine) => Number(machine.config?.metadata?.["alchemy.replica"])), |
| 255 | ).toEqual([...runningSlots]); |
| 256 | const roles = Array.from({ length: live.length }, (_, slot) => |
| 257 | runningSlots.includes(slot) ? "run" : "idle", |
| 258 | ).join(","); |
| 259 | for (const [slot, machine] of ordered.entries()) { |
| 260 | expect(machine.config?.metadata?.["alchemy.readiness-role"]).toBe( |
| 261 | runningSlots.includes(slot) ? "run" : "idle", |
| 262 | ); |
| 263 | expect(machine.config?.metadata?.["alchemy.readiness-roles"]).toBe(roles); |
| 264 | } |
| 265 | return ordered; |
| 266 | }; |
| 267 | |
| 268 | const assertTopology = ( |
| 269 | appName: string, |
| 270 | ids: string[], |
| 271 | mode: IdleMode, |
| 272 | runningSlots: readonly number[], |
| 273 | floor = 0, |
| 274 | mixed = false, |
| 275 | ) => |
| 276 | Effect.gen(function* () { |
| 277 | const live = yield* assertCommitted(appName, ids); |
| 278 | const ordered = assertSlots(live, runningSlots); |
| 279 | expect( |
| 280 | new Set(live.map((machine) => machine.config?.metadata?.["alchemy.generation"])).size, |
| 281 | ).toBe(1); |
| 282 | for (const [slot, machine] of ordered.entries()) { |
| 283 | expect(machine.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true"); |
| 284 | expect(machine.cordoned).toBe(false); |
| 285 | expect(machine.config?.services).toHaveLength(mixed ? 2 : 1); |
| 286 | expect(autostopMode(machine.config?.services?.[0]?.autostop)).toBe(mode); |
| 287 | expect(machine.config?.services?.[0]?.autostart).toBe(true); |
| 288 | expect(machine.config?.services?.[0]?.min_machines_running).toBe(floor); |
| 289 | if (mixed) { |
| 290 | expect(autostopMode(machine.config?.services?.[1]?.autostop)).toBe("off"); |
| 291 | } |
| 292 | if (!runningSlots.includes(slot)) { |
| 293 | expect(["created", "stopped", "suspended"]).toContain(machine.state); |
| 294 | } else { |
| 295 | expect(mixed ? ["started"] : ["started", "stopped", "suspended"]).toContain( |
| 296 | machine.state, |
| 297 | ); |
| 298 | expect(machine.instance_id).toBeDefined(); |
| 299 | expect(machine.config?.metadata?.["alchemy.checked-instance"]).toBe( |
| 300 | machine.instance_id, |
| 301 | ); |
| 302 | } |
| 303 | } |
| 304 | return ordered; |
| 305 | }); |
| 306 | |
| 307 | const checkedDeployment = ( |
| 308 | actor: Test.ScratchStack, |
| 309 | proxy: { events: TransportEvent[]; readiness: ReadinessEvent[] }, |
| 310 | version: string, |
| 311 | mode: IdleMode, |
| 312 | count: number, |
| 313 | runningSlots: readonly number[], |
| 314 | priorIds: readonly string[] = [], |
| 315 | floor = 0, |
| 316 | mixed = false, |
| 317 | ) => |
| 318 | Effect.gen(function* () { |
| 319 | const begin = proxy.events.length; |
| 320 | const proofBegin = proxy.readiness.length; |
| 321 | const output = yield* deployWorker( |
| 322 | actor, |
| 323 | version, |
| 324 | topology(mode, count, floor, mixed), |
| 325 | ).pipe(Effect.scoped); |
| 326 | const live = yield* assertTopology( |
| 327 | output.appName, |
| 328 | output.machineIds, |
| 329 | mode, |
| 330 | runningSlots, |
| 331 | floor, |
| 332 | mixed, |
| 333 | ); |
| 334 | assertReadinessCommit( |
| 335 | proxy.readiness.slice(proofBegin), |
| 336 | priorIds, |
| 337 | live, |
| 338 | runningSlots, |
| 339 | ["ready", "servicecheck-00-http-80"], |
| 340 | !mixed, |
| 341 | ); |
| 342 | expect( |
| 343 | proxy.events |
| 344 | .slice(begin) |
| 345 | .some( |
| 346 | (event) => |
| 347 | event.stage === "request" && |
| 348 | priorIds.includes(event.machineId!) && |
| 349 | event.path.endsWith("/start"), |
| 350 | ), |
| 351 | ).toBe(false); |
| 352 | return output; |
| 353 | }); |
| 354 | |
| 355 | // Each case deploys its own stack; these are the file's longest cases |
| 356 | // (3-8 min of real idle waits), so run them side by side. |
| 357 | describe.concurrent("live idle topology and automatic cadence", () => { |
| 358 | for (const mode of ["stop", "suspend"] as const) { |
| 359 | test.provider( |
| 360 | `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`, |
| 361 | (stack) => |
| 362 | Effect.gen(function* () { |
| 363 | yield* stack.destroy(); |
| 364 | const site = yield* stack.deploy(Fly.App("Site")); |
| 365 | try { |
| 366 | const proxy = yield* readinessProxy(); |
| 367 | const actor = yield* readinessActor( |
| 368 | stack, |
| 369 | `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`, |
| 370 | file, |
| 371 | proxy, |
| 372 | ); |
| 373 | const first = yield* checkedDeployment(actor, proxy, "one", mode, 1, [0]); |
| 374 | yield* idleAll(site.appName, first.machineIds, mode); |
| 375 | const up = yield* checkedDeployment( |
| 376 | actor, |
| 377 | proxy, |
| 378 | "two", |
| 379 | mode, |
| 380 | 3, |
| 381 | [0], |
| 382 | first.machineIds, |
| 383 | ); |
| 384 | expect(up.machineIds).toHaveLength(3); |
| 385 | expect(up.machineIds.every((id) => !first.machineIds.includes(id))).toBe(true); |
| 386 | yield* idleAll(site.appName, up.machineIds, mode); |
| 387 | const down = yield* checkedDeployment( |
| 388 | actor, |
| 389 | proxy, |
| 390 | "three", |
| 391 | mode, |
| 392 | 1, |
| 393 | [0], |
| 394 | up.machineIds, |
| 395 | ); |
| 396 | expect(down.machineIds).toHaveLength(1); |
| 397 | expect(up.machineIds).not.toContain(down.machineId); |
| 398 | yield* idleAll(site.appName, down.machineIds, mode); |
| 399 | const unchanged = yield* deployWorker(actor, "three", topology(mode, 1)); |
| 400 | expect(unchanged.machineIds).toEqual(down.machineIds); |
| 401 | expect((yield* census(site.appName))[0]!.state).toBe( |
| 402 | mode === "stop" ? "stopped" : "suspended", |
| 403 | ); |
| 404 | } finally { |
| 405 | yield* stack.destroy(); |
| 406 | yield* assertAppGone(site.appName); |
| 407 | } |
| 408 | }).pipe(Effect.scoped), |
| 409 | { timeout: 900_000 }, |
| 410 | ); |
| 411 | |
| 412 | test.provider( |
| 413 | `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`, |
| 414 | (stack) => |
| 415 | Effect.gen(function* () { |
| 416 | yield* stack.destroy(); |
| 417 | const site = yield* stack.deploy(Fly.App("Site")); |
| 418 | try { |
| 419 | const proxy = yield* readinessProxy(); |
| 420 | const actor = yield* readinessActor( |
| 421 | stack, |
| 422 | `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`, |
| 423 | file, |
| 424 | proxy, |
| 425 | ); |
| 426 | const first = yield* checkedDeployment(actor, proxy, "one", mode, 3, [0]); |
| 427 | yield* idleAll(site.appName, first.machineIds, mode); |
| 428 | const raised = yield* checkedDeployment( |
| 429 | actor, |
| 430 | proxy, |
| 431 | "one", |
| 432 | mode, |
| 433 | 3, |
| 434 | [0, 1], |
| 435 | first.machineIds, |
| 436 | 2, |
| 437 | ); |
| 438 | yield* idleAll(site.appName, raised.machineIds, mode); |
| 439 | const lowered = yield* checkedDeployment( |
| 440 | actor, |
| 441 | proxy, |
| 442 | "one", |
| 443 | mode, |
| 444 | 3, |
| 445 | [0], |
| 446 | raised.machineIds, |
| 447 | ); |
| 448 | yield* idleAll(site.appName, lowered.machineIds, mode); |
| 449 | const slots = assertSlots(yield* census(site.appName), [0]); |
| 450 | yield* machines.startMachine({ |
| 451 | app_name: site.appName, |
| 452 | machine_id: slots[2]!.id!, |
| 453 | }); |
| 454 | yield* waitState(site.appName, slots[2]!.id!, "started"); |
| 455 | const mixedOld = (yield* census(site.appName)).sort( |
| 456 | (a, b) => |
| 457 | Number(a.config?.metadata?.["alchemy.replica"]) - |
| 458 | Number(b.config?.metadata?.["alchemy.replica"]), |
| 459 | ); |
| 460 | expect(mixedOld.map((machine) => machine.state)).toEqual([ |
| 461 | mode === "stop" ? "stopped" : "suspended", |
| 462 | mode === "stop" ? "stopped" : "suspended", |
| 463 | "started", |
| 464 | ]); |
| 465 | const retained = yield* checkedDeployment( |
| 466 | actor, |
| 467 | proxy, |
| 468 | "retained", |
| 469 | mode, |
| 470 | 3, |
| 471 | [2], |
| 472 | lowered.machineIds, |
| 473 | ); |
| 474 | yield* idleAll(site.appName, retained.machineIds, mode); |
| 475 | const retainedSlots = assertSlots(yield* census(site.appName), [2]); |
| 476 | yield* machines.startMachine({ |
| 477 | app_name: site.appName, |
| 478 | machine_id: retainedSlots[2]!.id!, |
| 479 | }); |
| 480 | yield* waitState(site.appName, retainedSlots[2]!.id!, "started"); |
| 481 | expect( |
| 482 | assertSlots(yield* census(site.appName), [2]).map((machine) => machine.state), |
| 483 | ).toEqual([ |
| 484 | mode === "stop" ? "stopped" : "suspended", |
| 485 | mode === "stop" ? "stopped" : "suspended", |
| 486 | "started", |
| 487 | ]); |
| 488 | const retainedFloor = yield* checkedDeployment( |
| 489 | actor, |
| 490 | proxy, |
| 491 | "retained", |
| 492 | mode, |
| 493 | 3, |
| 494 | [0, 2], |
| 495 | retained.machineIds, |
| 496 | 2, |
| 497 | ); |
| 498 | yield* idleAll(site.appName, retainedFloor.machineIds, mode); |
| 499 | const mixed = yield* checkedDeployment( |
| 500 | actor, |
| 501 | proxy, |
| 502 | "one", |
| 503 | mode, |
| 504 | 3, |
| 505 | [0, 1, 2], |
| 506 | retainedFloor.machineIds, |
| 507 | 0, |
| 508 | true, |
| 509 | ); |
| 510 | yield* idleAll(site.appName, mixed.machineIds, mode); |
| 511 | yield* checkedDeployment(actor, proxy, "one", mode, 3, [0], mixed.machineIds); |
| 512 | } finally { |
| 513 | yield* stack.destroy(); |
| 514 | yield* assertAppGone(site.appName); |
| 515 | } |
| 516 | }).pipe(Effect.scoped), |
| 517 | { timeout: 1_200_000 }, |
| 518 | ); |
| 519 | |
| 520 | for (const recoveredCount of [3, 1, 4]) { |
| 521 | test.provider( |
| 522 | `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`, |
| 523 | (stack) => |
| 524 | Effect.gen(function* () { |
| 525 | yield* stack.destroy(); |
| 526 | const site = yield* stack.deploy(Fly.App("Site")); |
| 527 | try { |
| 528 | const proxy = yield* readinessProxy(); |
| 529 | const initial = yield* readinessActor( |
| 530 | stack, |
| 531 | `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`, |
| 532 | file, |
| 533 | proxy, |
| 534 | ); |
| 535 | const first = yield* checkedDeployment(initial, proxy, "one", mode, 3, [0]); |
| 536 | yield* idleAll(site.appName, first.machineIds, mode); |
| 537 | const source = (yield* census(site.appName))[0]!; |
| 538 | const metadata = source.config!.metadata!; |
| 539 | const attemptBegin = proxy.events.length; |
| 540 | yield* Effect.gen(function* () { |
| 541 | try { |
| 542 | const gate = yield* Deferred.make<string>(); |
| 543 | const actor = yield* readinessActor( |
| 544 | stack, |
| 545 | `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`, |
| 546 | file, |
| 547 | proxy, |
| 548 | (event) => |
| 549 | event.method === "POST" && |
| 550 | event.path === |
| 551 | `/v1/apps/${site.appName}/machines/${event.machineId}` && |
| 552 | event.metadata?.["alchemy.replica"] === "1" && |
| 553 | event.metadata["alchemy.fqn"] === metadata["alchemy.fqn"] && |
| 554 | event.metadata["alchemy.generation"] !== |
| 555 | metadata["alchemy.generation"] && |
| 556 | event.metadata["alchemy.readiness-roles"] === "run,run,idle" && |
| 557 | event.metadata["alchemy.idle-policy-restored"] === "true" && |
| 558 | event.phase === "promoting" |
| 559 | ? Deferred.succeed(gate, event.machineId!).pipe( |
| 560 | Effect.andThen(Effect.never), |
| 561 | ) |
| 562 | : Effect.void, |
| 563 | ); |
| 564 | const attempt = yield* deployWorker( |
| 565 | actor, |
| 566 | "two", |
| 567 | topology(mode, 3, 2), |
| 568 | ).pipe(Effect.scoped, Effect.forkScoped); |
| 569 | const barrierMachine = yield* Deferred.await(gate).pipe( |
| 570 | Effect.timeout("180 seconds"), |
| 571 | ); |
| 572 | const pending = yield* census(site.appName); |
| 573 | expect( |
| 574 | pending.filter((machine) => first.machineIds.includes(machine.id!)), |
| 575 | ).toHaveLength(3); |
| 576 | expect( |
| 577 | proxy.events.some((event) => retires(event, first.machineIds)), |
| 578 | ).toBe(false); |
| 579 | const candidates = yield* Effect.forEach( |
| 580 | pending.filter((machine) => !first.machineIds.includes(machine.id!)), |
| 581 | (machine) => |
| 582 | machines.getMachine({ |
| 583 | app_name: site.appName, |
| 584 | machine_id: machine.id!, |
| 585 | }), |
| 586 | ); |
| 587 | expect(candidates).toHaveLength(3); |
| 588 | expect( |
| 589 | candidates.every( |
| 590 | (machine) => |
| 591 | machine.config?.metadata?.["alchemy.phase"] === "promoting", |
| 592 | ), |
| 593 | ).toBe(true); |
| 594 | const pendingSlots = assertSlots(candidates, [0, 1]); |
| 595 | expect( |
| 596 | pendingSlots[0]!.config?.metadata?.["alchemy.idle-policy-restored"], |
| 597 | ).toBe("true"); |
| 598 | expect( |
| 599 | pendingSlots[1]!.config?.metadata?.["alchemy.idle-policy-restored"], |
| 600 | ).toBe("false"); |
| 601 | expect( |
| 602 | proxy.readiness.some( |
| 603 | (event) => |
| 604 | event.stage === "forwarded" && |
| 605 | event.method === "GET" && |
| 606 | event.machineId === pendingSlots[0]!.id && |
| 607 | event.instanceId === pendingSlots[0]!.instance_id && |
| 608 | event.metadata?.["alchemy.idle-policy-restored"] === "true" && |
| 609 | event.phase === "promoting" && |
| 610 | event.state === "started" && |
| 611 | readinessChecksPassing(event.checks, [ |
| 612 | "ready", |
| 613 | "servicecheck-00-http-80", |
| 614 | ]), |
| 615 | ), |
| 616 | ).toBe(true); |
| 617 | expect(barrierMachine).toBe(pendingSlots[1]!.id); |
| 618 | expect( |
| 619 | autostopMode(pendingSlots[0]!.config?.services?.[0]?.autostop), |
| 620 | ).toBe(mode); |
| 621 | expect( |
| 622 | autostopMode(pendingSlots[1]!.config?.services?.[0]?.autostop), |
| 623 | ).toBe("off"); |
| 624 | expect( |
| 625 | proxy.events |
| 626 | .slice(attemptBegin) |
| 627 | .some( |
| 628 | (event) => |
| 629 | event.method === "POST" && |
| 630 | event.path.endsWith(`/machines/${barrierMachine}`) && |
| 631 | event.phase === "promoting", |
| 632 | ), |
| 633 | ).toBe(false); |
| 634 | const read = yield* observeReplicaSet({ |
| 635 | appName: site.appName, |
| 636 | id: "Worker", |
| 637 | type: "Fly.Machine", |
| 638 | fqn: metadata["alchemy.fqn"]!, |
| 639 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 640 | baseName: first.baseName, |
| 641 | machineIds: first.machineIds, |
| 642 | }); |
| 643 | expect(read?.machineIds).toEqual(first.machineIds); |
| 644 | expect(read?.count).toBe(3); |
| 645 | expect(read?.rolloutPending).toBe(true); |
| 646 | const interruption = yield* Fiber.interrupt(attempt).pipe( |
| 647 | Effect.forkScoped, |
| 648 | ); |
| 649 | yield* Effect.yieldNow; |
| 650 | yield* Effect.sync(() => { |
| 651 | proxy.clear(); |
| 652 | proxy.release(); |
| 653 | }); |
| 654 | yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds")); |
| 655 | expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true); |
| 656 | expect( |
| 657 | proxy.events |
| 658 | .slice(attemptBegin) |
| 659 | .some( |
| 660 | (event) => |
| 661 | first.machineIds.includes(event.machineId!) && |
| 662 | event.path.endsWith("/start"), |
| 663 | ), |
| 664 | ).toBe(false); |
| 665 | expect( |
| 666 | proxy.readiness.some( |
| 667 | (event) => |
| 668 | event.stage === "request" && |
| 669 | event.path.endsWith("/metadata") && |
| 670 | event.phase === "active" && |
| 671 | candidates.some((machine) => machine.id === event.machineId), |
| 672 | ), |
| 673 | ).toBe(false); |
| 674 | expect( |
| 675 | proxy.events.some((event) => retires(event, first.machineIds)), |
| 676 | ).toBe(false); |
| 677 | const resumed = yield* readinessActor( |
| 678 | stack, |
| 679 | `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`, |
| 680 | file, |
| 681 | proxy, |
| 682 | ); |
| 683 | const begin = proxy.readiness.length; |
| 684 | const recovered = yield* deployWorker( |
| 685 | resumed, |
| 686 | "two", |
| 687 | topology(mode, recoveredCount, 2), |
| 688 | ).pipe(Effect.scoped); |
| 689 | expect(recovered.machineIds).toHaveLength(recoveredCount); |
| 690 | if (recoveredCount === 3) { |
| 691 | expect([...recovered.machineIds].sort()).toEqual( |
| 692 | candidates.map((machine) => machine.id).sort(), |
| 693 | ); |
| 694 | } else { |
| 695 | expect( |
| 696 | recovered.machineIds.every( |
| 697 | (id) => !pending.some((machine) => machine.id === id), |
| 698 | ), |
| 699 | ).toBe(true); |
| 700 | } |
| 701 | const runningSlots = recoveredCount === 1 ? [0] : [0, 1]; |
| 702 | const live = yield* assertTopology( |
| 703 | site.appName, |
| 704 | recovered.machineIds, |
| 705 | mode, |
| 706 | runningSlots, |
| 707 | 2, |
| 708 | ); |
| 709 | assertReadinessCommit( |
| 710 | proxy.readiness.slice(begin), |
| 711 | pending |
| 712 | .map((machine) => machine.id!) |
| 713 | .filter((id) => !recovered.machineIds.includes(id)), |
| 714 | live, |
| 715 | runningSlots, |
| 716 | ["ready", "servicecheck-00-http-80"], |
| 717 | true, |
| 718 | ); |
| 719 | const unchanged = yield* deployWorker( |
| 720 | resumed, |
| 721 | "two", |
| 722 | topology(mode, recoveredCount, 2), |
| 723 | ).pipe(Effect.scoped); |
| 724 | expect(unchanged.machineIds).toEqual(recovered.machineIds); |
| 725 | const committed = yield* observeReplicaSet({ |
| 726 | appName: site.appName, |
| 727 | id: "Worker", |
| 728 | type: "Fly.Machine", |
| 729 | fqn: metadata["alchemy.fqn"]!, |
| 730 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 731 | machineIds: recovered.machineIds, |
| 732 | }); |
| 733 | expect(committed?.rolloutPending).toBe(false); |
| 734 | expect(committed?.machineIds).toEqual(recovered.machineIds); |
| 735 | } finally { |
| 736 | yield* Effect.sync(() => { |
| 737 | proxy.clear(); |
| 738 | proxy.release(); |
| 739 | }); |
| 740 | } |
| 741 | }).pipe(Effect.scoped); |
| 742 | } finally { |
| 743 | yield* stack.destroy(); |
| 744 | yield* assertAppGone(site.appName); |
| 745 | } |
| 746 | }).pipe(Effect.scoped), |
| 747 | { timeout: 900_000 }, |
| 748 | ); |
| 749 | } |
| 750 | |
| 751 | test.provider( |
| 752 | `S11 public traffic autostarts ${mode} and Fly automatically returns it to idle within 15 minutes`, |
| 753 | (stack) => |
| 754 | Effect.gen(function* () { |
| 755 | yield* stack.destroy(); |
| 756 | const site = yield* stack.deploy(Fly.App("Site")); |
| 757 | try { |
| 758 | const output = yield* stack.deploy( |
| 759 | Effect.gen(function* () { |
| 760 | const app = yield* Fly.App("Site"); |
| 761 | yield* Fly.IpAssignment("Public", { app, type: "shared_v4" }); |
| 762 | return yield* Fly.Machine("Worker", { |
| 763 | app, |
| 764 | image: "nginx:alpine", |
| 765 | deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" }, |
| 766 | shutdown: { signal: "SIGQUIT", timeout: "5 seconds" }, |
| 767 | services: [ |
| 768 | { |
| 769 | protocol: "tcp", |
| 770 | internalPort: 80, |
| 771 | autostop: mode, |
| 772 | autostart: true, |
| 773 | minMachinesRunning: 0, |
| 774 | ports: [{ port: 80, handlers: ["http"] }], |
| 775 | checks: [checks.ready], |
| 776 | }, |
| 777 | ], |
| 778 | }); |
| 779 | }), |
| 780 | ); |
| 781 | yield* idleAll(site.appName, output.machineIds, mode); |
| 782 | const client = yield* HttpClient.HttpClient; |
| 783 | yield* Effect.gen(function* () { |
| 784 | const response = yield* client.get(`http://${site.appName}.fly.dev`); |
| 785 | const body = yield* response.text; |
| 786 | if (response.status !== 200 || !body.includes("Welcome to nginx")) { |
| 787 | return yield* Effect.fail( |
| 788 | new Error(`Public Fly route is not ready: ${response.status}`), |
| 789 | ); |
| 790 | } |
| 791 | expect(response.status).toBe(200); |
| 792 | expect(body).toContain("Welcome to nginx"); |
| 793 | }).pipe( |
| 794 | Effect.retry({ schedule: Schedule.spaced("3 seconds"), times: 20 }), |
| 795 | Effect.timeout("90 seconds"), |
| 796 | ); |
| 797 | yield* waitState(site.appName, output.machineId, "started"); |
| 798 | const started = yield* Clock.currentTimeMillis; |
| 799 | const idleState = mode === "stop" ? "stopped" : "suspended"; |
| 800 | // Only native GETs follow the last public request; no stop, suspend, or update substitutes. |
| 801 | const idle = yield* machines |
| 802 | .getMachine({ app_name: site.appName, machine_id: output.machineId }) |
| 803 | .pipe( |
| 804 | Effect.tap((machine) => |
| 805 | Effect.logInfo("Automatic Fly idle observation", { |
| 806 | mode, |
| 807 | state: machine.state, |
| 808 | instanceId: machine.instance_id, |
| 809 | }), |
| 810 | ), |
| 811 | Effect.repeat({ |
| 812 | schedule: Schedule.spaced("10 seconds"), |
| 813 | times: 90, |
| 814 | until: (machine) => machine.state === idleState, |
| 815 | }), |
| 816 | Effect.timeout("15 minutes"), |
| 817 | ); |
| 818 | expect(idle.state).toBe(idleState); |
| 819 | expect(idle.cordoned).toBe(false); |
| 820 | expect(autostopMode(idle.config?.services?.[0]?.autostop)).toBe(mode); |
| 821 | expect(idle.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true"); |
| 822 | yield* Effect.logInfo("Automatic return to idle observed", { |
| 823 | mode, |
| 824 | elapsedMs: (yield* Clock.currentTimeMillis) - started, |
| 825 | }); |
| 826 | expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual( |
| 827 | output.machineIds, |
| 828 | ); |
| 829 | } finally { |
| 830 | yield* stack.destroy(); |
| 831 | yield* assertAppGone(site.appName); |
| 832 | } |
| 833 | }), |
| 834 | { tags: ["provider:fly:ipassignment"], timeout: 1_200_000 }, |
| 835 | ); |
| 836 | } |
| 837 | }); |
| 838 | }, |
| 839 | ); |
| 840 | |
| 841 | describe.sequential( |
| 842 | "idle capacity", |
| 843 | { |
| 844 | tags: [ |
| 845 | "provider:fly", |
| 846 | "provider:fly:app", |
| 847 | "provider:fly:machine", |
| 848 | "provider:fly:service", |
| 849 | "live", |
| 850 | ], |
| 851 | }, |
| 852 | () => { |
| 853 | let endpoint: string | undefined; |
| 854 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 855 | |
| 856 | describe.sequential("idle topology", () => { |
| 857 | for (const mode of ["stop", "suspend"] as const) { |
| 858 | for (const allIdle of [false, true]) { |
| 859 | test.provider( |
| 860 | `S05 S11 ${mode} ${allIdle ? "all-idle" : "mixed"} replacement keeps an idle nonrepresentative`, |
| 861 | (stack) => |
| 862 | Effect.gen(function* () { |
| 863 | yield* stack.destroy(); |
| 864 | const props = { |
| 865 | count: 2, |
| 866 | services: [ |
| 867 | { |
| 868 | protocol: "tcp", |
| 869 | internalPort: 80, |
| 870 | autostop: mode, |
| 871 | autostart: true, |
| 872 | minMachinesRunning: 0, |
| 873 | ports: [{ port: 80, handlers: ["http"] }], |
| 874 | checks: [checks.ready], |
| 875 | }, |
| 876 | ], |
| 877 | }; |
| 878 | const initial = yield* deployWorker(stack, "one", { |
| 879 | ...props, |
| 880 | deploy: { strategy: "rolling" }, |
| 881 | }); |
| 882 | for (const id of allIdle ? initial.machineIds : initial.machineIds.slice(1)) { |
| 883 | const target = { app_name: initial.appName, machine_id: id }; |
| 884 | yield* mode === "stop" |
| 885 | ? machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" }) |
| 886 | : machines.suspendMachine(target); |
| 887 | const observed = yield* machines.getMachine(target).pipe( |
| 888 | Effect.repeat({ |
| 889 | schedule: Schedule.spaced("2 seconds"), |
| 890 | times: 8, |
| 891 | until: (machine) => |
| 892 | machine.state === (mode === "stop" ? "stopped" : "suspended"), |
| 893 | }), |
| 894 | ); |
| 895 | expect(observed.state).toBe(mode === "stop" ? "stopped" : "suspended"); |
| 896 | } |
| 897 | const proxy = yield* transportProxy(); |
| 898 | yield* Effect.sync(() => { |
| 899 | endpoint = proxy.url; |
| 900 | }); |
| 901 | const next = yield* deployWorker(stack, "two", props); |
| 902 | expect(next.machineIds).toHaveLength(2); |
| 903 | const live = yield* census(initial.appName); |
| 904 | expect(live.map((machine) => machine.id).sort()).toEqual( |
| 905 | [...next.machineIds].sort(), |
| 906 | ); |
| 907 | expect( |
| 908 | live.every((machine) => { |
| 909 | const autostop = machine.config?.services?.[0]?.autostop; |
| 910 | return (autostop === true ? "stop" : autostop) === mode; |
| 911 | }), |
| 912 | ).toBe(true); |
| 913 | expect( |
| 914 | live.filter((machine) => machine.state === "started").length, |
| 915 | ).toBeLessThanOrEqual(1); |
| 916 | expect( |
| 917 | live.some((machine) => ["stopped", "suspended"].includes(machine.state!)), |
| 918 | ).toBe(true); |
| 919 | expect( |
| 920 | proxy.events.some( |
| 921 | (event) => |
| 922 | initial.machineIds.includes(event.machineId!) && |
| 923 | event.path.endsWith("/start"), |
| 924 | ), |
| 925 | ).toBe(false); |
| 926 | const same = yield* deployWorker(stack, "two", props); |
| 927 | expect(same.machineIds).toEqual(next.machineIds); |
| 928 | yield* Effect.sync(() => { |
| 929 | endpoint = undefined; |
| 930 | }); |
| 931 | yield* stack.destroy(); |
| 932 | yield* assertAppGone(initial.appName); |
| 933 | }).pipe( |
| 934 | Effect.scoped, |
| 935 | Effect.ensuring( |
| 936 | Effect.sync(() => { |
| 937 | endpoint = undefined; |
| 938 | }), |
| 939 | ), |
| 940 | ), |
| 941 | { timeout: 300_000 }, |
| 942 | ); |
| 943 | } |
| 944 | } |
| 945 | }); |
| 946 | }, |
| 947 | ); |
| 948 | |
| 949 | describe.sequential( |
| 950 | "post-promotion health", |
| 951 | { |
| 952 | tags: [ |
| 953 | "provider:fly", |
| 954 | "provider:fly:app", |
| 955 | "provider:fly:ipassignment", |
| 956 | "provider:fly:machine", |
| 957 | "provider:fly:secret", |
| 958 | "provider:fly:service", |
| 959 | "live", |
| 960 | ], |
| 961 | }, |
| 962 | () => { |
| 963 | let endpoint: string | undefined; |
| 964 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 965 | |
| 966 | describe.sequential("post-promotion health", () => { |
| 967 | for (const changed of [false, true]) { |
| 968 | test.provider( |
| 969 | `F09 a real post-uncordon health flip preserves both generations then recovers ${changed ? "changed" : "same"} desired state`, |
| 970 | (stack) => |
| 971 | Effect.gen(function* () { |
| 972 | yield* stack.destroy(); |
| 973 | const readiness = yield* makeReadinessControl(); |
| 974 | const initial = yield* readiness.deployWorker(stack, "one"); |
| 975 | const proxy = yield* transportProxy(); |
| 976 | yield* Effect.sync(() => { |
| 977 | endpoint = proxy.url; |
| 978 | proxy.arm({ |
| 979 | match: (event) => event.path.endsWith("/uncordon"), |
| 980 | action: "hold-response", |
| 981 | remaining: 1, |
| 982 | }); |
| 983 | }); |
| 984 | const update = yield* readiness |
| 985 | .deployWorker(stack, "two") |
| 986 | .pipe(Effect.result, Effect.forkScoped); |
| 987 | const promoted = yield* proxy.wait( |
| 988 | (event) => |
| 989 | event.stage === "held" && |
| 990 | event.path.endsWith("/uncordon") && |
| 991 | event.status! < 300, |
| 992 | ); |
| 993 | expect(promoted.machineId).toBeDefined(); |
| 994 | yield* readiness.turnOff(initial.appName, promoted.machineId!); |
| 995 | yield* Effect.sync(proxy.release); |
| 996 | const result = yield* Fiber.join(update).pipe(Effect.timeout("60 seconds")); |
| 997 | expect(Result.isFailure(result)).toBe(true); |
| 998 | if (Result.isFailure(result)) |
| 999 | expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" }); |
| 1000 | const live = yield* census(initial.appName); |
| 1001 | expect(live.map((machine) => machine.id).sort()).toEqual( |
| 1002 | [initial.machineId, promoted.machineId!].sort(), |
| 1003 | ); |
| 1004 | expect(live.every((machine) => machine.cordoned === false)).toBe(true); |
| 1005 | const pending = live.find((machine) => machine.id === promoted.machineId)!; |
| 1006 | const metadata = pending.config!.metadata!; |
| 1007 | expect(metadata["alchemy.phase"]).toBe("validating"); |
| 1008 | expect(metadata["alchemy.checked-instance"]).toBeUndefined(); |
| 1009 | const read = yield* observeReplicaSet({ |
| 1010 | appName: initial.appName, |
| 1011 | id: "Worker", |
| 1012 | type: "Fly.Machine", |
| 1013 | fqn: metadata["alchemy.fqn"]!, |
| 1014 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 1015 | machineIds: initial.machineIds, |
| 1016 | }); |
| 1017 | expect(read?.machineIds).toEqual(initial.machineIds); |
| 1018 | expect(read?.rolloutPending).toBe(true); |
| 1019 | expect( |
| 1020 | proxy.events.some( |
| 1021 | (event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path), |
| 1022 | ), |
| 1023 | ).toBe(false); |
| 1024 | if (!changed) { |
| 1025 | yield* repairReadiness(initial.appName, promoted.machineId!); |
| 1026 | } |
| 1027 | const recovered = yield* readiness.deployWorker(stack, changed ? "three" : "two"); |
| 1028 | if (changed) expect(recovered.machineId).not.toBe(promoted.machineId); |
| 1029 | else expect(recovered.machineId).toBe(promoted.machineId); |
| 1030 | yield* assertCommitted(initial.appName, recovered.machineIds); |
| 1031 | yield* Effect.sync(() => { |
| 1032 | endpoint = undefined; |
| 1033 | }); |
| 1034 | yield* stack.destroy(); |
| 1035 | yield* assertAppGone(initial.appName); |
| 1036 | }).pipe( |
| 1037 | Effect.scoped, |
| 1038 | Effect.ensuring( |
| 1039 | Effect.sync(() => { |
| 1040 | endpoint = undefined; |
| 1041 | }), |
| 1042 | ), |
| 1043 | ), |
| 1044 | { timeout: 300_000 }, |
| 1045 | ); |
| 1046 | } |
| 1047 | }); |
| 1048 | }, |
| 1049 | ); |
| 1050 | |
| 1051 | describe.sequential( |
| 1052 | "ownership", |
| 1053 | { |
| 1054 | tags: [ |
| 1055 | "provider:fly", |
| 1056 | "provider:fly:app", |
| 1057 | "provider:fly:machine", |
| 1058 | "provider:fly:service", |
| 1059 | "live", |
| 1060 | ], |
| 1061 | }, |
| 1062 | () => { |
| 1063 | const { test } = Test.make({ providers: Fly.providers() }); |
| 1064 | |
| 1065 | test.provider( |
| 1066 | "S09 foreign routed Machine and sibling survive rollout and stale-instance deletion", |
| 1067 | (stack) => |
| 1068 | Effect.gen(function* () { |
| 1069 | yield* stack.destroy(); |
| 1070 | const deploy = (version: string) => |
| 1071 | stack.deploy( |
| 1072 | Effect.gen(function* () { |
| 1073 | const app = yield* Fly.App("Site"); |
| 1074 | const sibling = yield* Fly.Machine("Sibling", { |
| 1075 | app, |
| 1076 | image: "nginx:alpine", |
| 1077 | env: { VERSION: "sibling" }, |
| 1078 | }); |
| 1079 | const worker = yield* Fly.Machine("Worker", { |
| 1080 | app, |
| 1081 | image: "nginx:alpine", |
| 1082 | env: { VERSION: version }, |
| 1083 | checks, |
| 1084 | deploy: { strategy: "bluegreen", healthTimeout: "20 seconds" }, |
| 1085 | shutdown: { signal: "SIGQUIT", timeout: "5 seconds" }, |
| 1086 | }); |
| 1087 | return { sibling, worker }; |
| 1088 | }), |
| 1089 | ); |
| 1090 | const initial = yield* deploy("one"); |
| 1091 | const foreign = yield* machines.createMachine({ |
| 1092 | app_name: initial.worker.appName, |
| 1093 | name: "foreign-routed", |
| 1094 | region: "iad", |
| 1095 | config: { |
| 1096 | image: "nginx:alpine", |
| 1097 | env: { VERSION: "foreign" }, |
| 1098 | services: [ |
| 1099 | { |
| 1100 | protocol: "tcp", |
| 1101 | internal_port: 80, |
| 1102 | ports: [{ port: 80, handlers: ["http"] }], |
| 1103 | }, |
| 1104 | ], |
| 1105 | }, |
| 1106 | }); |
| 1107 | const removeForeign = machines |
| 1108 | .deleteMachine({ |
| 1109 | app_name: initial.worker.appName, |
| 1110 | machine_id: foreign.id!, |
| 1111 | force: true, |
| 1112 | }) |
| 1113 | .pipe(Effect.catchTag("NotFound", () => Effect.void)); |
| 1114 | yield* Effect.gen(function* () { |
| 1115 | const next = yield* deploy("two"); |
| 1116 | expect(next.worker.machineId).not.toBe(initial.worker.machineId); |
| 1117 | expect(next.sibling.machineId).toBe(initial.sibling.machineId); |
| 1118 | const before = yield* census(initial.worker.appName); |
| 1119 | expect(before.map((machine) => machine.id).sort()).toEqual( |
| 1120 | [next.worker.machineId, next.sibling.machineId, foreign.id!].sort(), |
| 1121 | ); |
| 1122 | expect(before.find((machine) => machine.id === foreign.id)?.cordoned).toBe(false); |
| 1123 | const owned = before.find((machine) => machine.id === next.worker.machineId)!; |
| 1124 | yield* deleteReplicaSet({ |
| 1125 | appName: next.worker.appName, |
| 1126 | id: "Worker", |
| 1127 | type: "Fly.Machine", |
| 1128 | fqn: owned.config!.metadata!["alchemy.fqn"]!, |
| 1129 | resourceInstanceId: "stale-instance-must-not-delete-successor", |
| 1130 | machineIds: next.worker.machineIds, |
| 1131 | volumeIds: [], |
| 1132 | }); |
| 1133 | expect( |
| 1134 | (yield* census(initial.worker.appName)).map((machine) => machine.id).sort(), |
| 1135 | ).toEqual(before.map((machine) => machine.id).sort()); |
| 1136 | }).pipe(Effect.ensuring(removeForeign.pipe(Effect.orDie))); |
| 1137 | yield* stack.destroy(); |
| 1138 | yield* assertAppGone(initial.worker.appName); |
| 1139 | }), |
| 1140 | { timeout: 300_000 }, |
| 1141 | ); |
| 1142 | |
| 1143 | test.provider( |
| 1144 | "S09 F11 stale output and a same-logical-ID foreign FQN cannot authorize successor deletion", |
| 1145 | (stack) => |
| 1146 | Effect.gen(function* () { |
| 1147 | yield* stack.destroy(); |
| 1148 | const deploy = (name: string, version: string) => |
| 1149 | stack.deploy( |
| 1150 | Effect.gen(function* () { |
| 1151 | const app = yield* Fly.App("Site"); |
| 1152 | return yield* Fly.Machine("Worker", { |
| 1153 | app, |
| 1154 | name, |
| 1155 | image: "nginx:alpine", |
| 1156 | checks, |
| 1157 | env: { VERSION: version }, |
| 1158 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 1159 | shutdown: { signal: "SIGQUIT", timeout: "5 seconds" }, |
| 1160 | }); |
| 1161 | }), |
| 1162 | ); |
| 1163 | const initial = yield* deploy("stale-original", "one"); |
| 1164 | const old = yield* machines.getMachine({ |
| 1165 | app_name: initial.appName, |
| 1166 | machine_id: initial.machineId, |
| 1167 | }); |
| 1168 | const oldMetadata = old.config!.metadata!; |
| 1169 | const collision = yield* machines.createMachine({ |
| 1170 | app_name: initial.appName, |
| 1171 | name: "fqn-collision", |
| 1172 | region: "iad", |
| 1173 | config: { |
| 1174 | ...old.config, |
| 1175 | metadata: { |
| 1176 | ...oldMetadata, |
| 1177 | "alchemy.fqn": `${oldMetadata["alchemy.fqn"]}/OtherScope`, |
| 1178 | }, |
| 1179 | }, |
| 1180 | }); |
| 1181 | yield* Effect.gen(function* () { |
| 1182 | const next = yield* deploy("stale-original", "two"); |
| 1183 | const refreshed = yield* observeReplicaSet({ |
| 1184 | appName: initial.appName, |
| 1185 | id: "Worker", |
| 1186 | type: "Fly.Machine", |
| 1187 | fqn: oldMetadata["alchemy.fqn"]!, |
| 1188 | resourceInstanceId: oldMetadata["alchemy.instance"]!, |
| 1189 | baseName: initial.baseName, |
| 1190 | machineIds: [...initial.machineIds, collision.id!], |
| 1191 | }); |
| 1192 | expect(refreshed?.machineIds).toEqual(next.machineIds); |
| 1193 | expect(refreshed?.rolloutPending).toBe(false); |
| 1194 | const successor = yield* deploy("stale-successor", "three"); |
| 1195 | const current = yield* machines.getMachine({ |
| 1196 | app_name: initial.appName, |
| 1197 | machine_id: successor.machineId, |
| 1198 | }); |
| 1199 | expect(current.config?.metadata?.["alchemy.instance"]).not.toBe( |
| 1200 | oldMetadata["alchemy.instance"], |
| 1201 | ); |
| 1202 | // Even an ID cache containing the new Machine grants no old-lineage authority. |
| 1203 | yield* deleteReplicaSet({ |
| 1204 | appName: initial.appName, |
| 1205 | id: "Worker", |
| 1206 | type: "Fly.Machine", |
| 1207 | fqn: oldMetadata["alchemy.fqn"]!, |
| 1208 | resourceInstanceId: oldMetadata["alchemy.instance"]!, |
| 1209 | machineIds: [ |
| 1210 | ...initial.machineIds, |
| 1211 | ...next.machineIds, |
| 1212 | ...successor.machineIds, |
| 1213 | collision.id!, |
| 1214 | ], |
| 1215 | volumeIds: [], |
| 1216 | }); |
| 1217 | const live = yield* census(initial.appName); |
| 1218 | expect(live.map((machine) => machine.id).sort()).toEqual( |
| 1219 | [successor.machineId, collision.id!].sort(), |
| 1220 | ); |
| 1221 | const foreign = live.find((machine) => machine.id === collision.id)!; |
| 1222 | expect(foreign.config?.metadata?.["alchemy.fqn"]).toBe( |
| 1223 | `${oldMetadata["alchemy.fqn"]}/OtherScope`, |
| 1224 | ); |
| 1225 | expect(foreign.cordoned).toBe(false); |
| 1226 | }).pipe( |
| 1227 | Effect.ensuring( |
| 1228 | machines |
| 1229 | .deleteMachine({ |
| 1230 | app_name: initial.appName, |
| 1231 | machine_id: collision.id!, |
| 1232 | force: true, |
| 1233 | }) |
| 1234 | .pipe( |
| 1235 | Effect.catchTag("NotFound", () => Effect.void), |
| 1236 | Effect.orDie, |
| 1237 | ), |
| 1238 | ), |
| 1239 | ); |
| 1240 | yield* stack.destroy(); |
| 1241 | yield* assertAppGone(initial.appName); |
| 1242 | }), |
| 1243 | { timeout: 600_000 }, |
| 1244 | ); |
| 1245 | }, |
| 1246 | ); |
| 1247 | |
| 1248 | describe.sequential( |
| 1249 | "deployment", |
| 1250 | { |
| 1251 | tags: [ |
| 1252 | "provider:fly", |
| 1253 | "provider:fly:app", |
| 1254 | "provider:fly:machine", |
| 1255 | "provider:fly:service", |
| 1256 | "live", |
| 1257 | ], |
| 1258 | }, |
| 1259 | () => { |
| 1260 | const { test } = Test.make({ providers: Fly.providers() }); |
| 1261 | |
| 1262 | test.provider( |
| 1263 | "S01 bluegreen worker checks, promotion, replacement, and graceful teardown", |
| 1264 | (stack) => |
| 1265 | Effect.gen(function* () { |
| 1266 | yield* stack.destroy(); |
| 1267 | const deploy = (version: string, path = "/") => |
| 1268 | stack.deploy( |
| 1269 | Effect.gen(function* () { |
| 1270 | const app = yield* Fly.App("Site"); |
| 1271 | return yield* Fly.Machine("Worker", { |
| 1272 | app, |
| 1273 | name: "bluegreen-worker-long-name-base", |
| 1274 | image: "nginx:alpine", |
| 1275 | env: { VERSION: version }, |
| 1276 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 1277 | shutdown: { signal: "SIGQUIT", timeout: "10 seconds" }, |
| 1278 | checks: { |
| 1279 | ready: { type: "http", port: 80, path, interval: "2s", timeout: "1s" }, |
| 1280 | }, |
| 1281 | }); |
| 1282 | }), |
| 1283 | ); |
| 1284 | const first = yield* deploy("one"); |
| 1285 | const committed = yield* machines.getMachine({ |
| 1286 | app_name: first.appName, |
| 1287 | machine_id: first.machineId, |
| 1288 | }); |
| 1289 | // Active commit metadata can reset reports after the provider's readiness validation. |
| 1290 | const machine = yield* waitHealthy(first.appName, committed, 30_000); |
| 1291 | expect(machine.instance_id).toBe(committed.instance_id); |
| 1292 | yield* Effect.logInfo("Observed promoted worker", { |
| 1293 | checks: machine.checks?.map(({ name, status }) => ({ name, status })), |
| 1294 | configured: machine.config?.checks, |
| 1295 | metadata: machine.config?.metadata, |
| 1296 | }); |
| 1297 | expect(machine.cordoned).toBe(false); |
| 1298 | expect( |
| 1299 | machine.checks?.some( |
| 1300 | (check) => check.name === "ready" && check.status === "passing", |
| 1301 | ), |
| 1302 | ).toBe(true); |
| 1303 | expect(machine.config?.metadata?.["alchemy.phase"]).toBe("active"); |
| 1304 | expect(first.name.length).toBeLessThanOrEqual(30); |
| 1305 | const second = yield* deploy("two"); |
| 1306 | expect(second.machineId).not.toBe(first.machineId); |
| 1307 | expect(second.baseName).toBe(first.baseName); |
| 1308 | const listed = yield* machines.listMachines({ app_name: first.appName }); |
| 1309 | expect( |
| 1310 | listed |
| 1311 | .filter((machine) => machine.state !== "destroyed") |
| 1312 | .map((machine) => machine.id), |
| 1313 | ).toEqual([second.machineId]); |
| 1314 | yield* stack.destroy(); |
| 1315 | yield* assertAppGone(first.appName); |
| 1316 | }), |
| 1317 | { timeout: 180_000 }, |
| 1318 | ); |
| 1319 | |
| 1320 | test.provider( |
| 1321 | "S04 unhealthy replacement preserves the old routed ID and cleans unpromoted candidates", |
| 1322 | (stack) => |
| 1323 | Effect.gen(function* () { |
| 1324 | yield* stack.destroy(); |
| 1325 | const initial = yield* deployWorker(stack, "one"); |
| 1326 | const failed = yield* deployWorker(stack, "broken", { |
| 1327 | checks: { ready: { ...checks.ready, path: "/missing" } }, |
| 1328 | deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" }, |
| 1329 | }).pipe(Effect.result); |
| 1330 | expect(Result.isFailure(failed)).toBe(true); |
| 1331 | if (Result.isFailure(failed)) |
| 1332 | expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" }); |
| 1333 | const live = yield* census(initial.appName); |
| 1334 | expect(live.map((machine) => machine.id)).toEqual(initial.machineIds); |
| 1335 | expect(live[0]!.state).toBe("started"); |
| 1336 | expect(live[0]!.cordoned).toBe(false); |
| 1337 | yield* stack.destroy(); |
| 1338 | yield* assertAppGone(initial.appName); |
| 1339 | }), |
| 1340 | { timeout: 180_000 }, |
| 1341 | ); |
| 1342 | }, |
| 1343 | ); |
| 1344 | |
| 1345 | describe.sequential( |
| 1346 | "promotion faults", |
| 1347 | { |
| 1348 | tags: [ |
| 1349 | "provider:fly", |
| 1350 | "provider:fly:app", |
| 1351 | "provider:fly:machine", |
| 1352 | "provider:fly:service", |
| 1353 | "live", |
| 1354 | ], |
| 1355 | }, |
| 1356 | () => { |
| 1357 | let endpoint: string | undefined; |
| 1358 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 1359 | |
| 1360 | test.provider( |
| 1361 | "F03 F08 partial promotion loses the first uncordon response without rolling back serving green", |
| 1362 | (stack) => |
| 1363 | Effect.gen(function* () { |
| 1364 | yield* stack.destroy(); |
| 1365 | const initial = yield* deployWorker(stack, "one", { count: 2 }); |
| 1366 | const proxy = yield* transportProxy(); |
| 1367 | const match = (event: { path: string }) => event.path.endsWith("/uncordon"); |
| 1368 | yield* Effect.sync(() => { |
| 1369 | endpoint = proxy.url; |
| 1370 | proxy.arm({ match, action: "drop-response", remaining: 1 }); |
| 1371 | proxy.arm({ match, action: "cut-request", remaining: Infinity }); |
| 1372 | }); |
| 1373 | try { |
| 1374 | const failed = yield* deployWorker(stack, "two", { count: 2 }).pipe( |
| 1375 | Effect.timeout("120 seconds"), |
| 1376 | Effect.result, |
| 1377 | ); |
| 1378 | expect(Result.isFailure(failed)).toBe(true); |
| 1379 | const lost = proxy.events.find( |
| 1380 | (event) => event.stage === "dropped" && event.path.endsWith("/uncordon"), |
| 1381 | ); |
| 1382 | expect(lost?.status).toBeGreaterThanOrEqual(200); |
| 1383 | expect(lost?.status).toBeLessThan(300); |
| 1384 | const live = yield* census(initial.appName); |
| 1385 | expect( |
| 1386 | initial.machineIds.every((id) => |
| 1387 | live.some((machine) => machine.id === id && machine.cordoned === false), |
| 1388 | ), |
| 1389 | ).toBe(true); |
| 1390 | expect(live.find((machine) => machine.id === lost!.machineId)?.cordoned).toBe( |
| 1391 | false, |
| 1392 | ); |
| 1393 | const candidateIds = live |
| 1394 | .filter((machine) => !initial.machineIds.includes(machine.id!)) |
| 1395 | .map((machine) => machine.id!); |
| 1396 | expect(candidateIds).toHaveLength(2); |
| 1397 | expect( |
| 1398 | proxy.events.some( |
| 1399 | (event) => |
| 1400 | event.method === "DELETE" && |
| 1401 | /\/machines\/[^/]+$/.test(event.path) && |
| 1402 | candidateIds.includes(event.machineId!), |
| 1403 | ), |
| 1404 | ).toBe(false); |
| 1405 | yield* Effect.sync(proxy.clear); |
| 1406 | const recovered = yield* deployWorker(stack, "two", { count: 2 }); |
| 1407 | expect([...recovered.machineIds].sort()).toEqual(candidateIds.sort()); |
| 1408 | yield* assertCommitted(initial.appName, recovered.machineIds); |
| 1409 | } finally { |
| 1410 | yield* Effect.sync(() => { |
| 1411 | endpoint = undefined; |
| 1412 | proxy.clear(); |
| 1413 | }); |
| 1414 | } |
| 1415 | yield* stack.destroy(); |
| 1416 | yield* assertAppGone(initial.appName); |
| 1417 | }).pipe( |
| 1418 | Effect.scoped, |
| 1419 | Effect.ensuring( |
| 1420 | Effect.sync(() => { |
| 1421 | endpoint = undefined; |
| 1422 | }), |
| 1423 | ), |
| 1424 | ), |
| 1425 | { timeout: 300_000 }, |
| 1426 | ); |
| 1427 | }, |
| 1428 | ); |
| 1429 | |
| 1430 | describe.sequential( |
| 1431 | "transport", |
| 1432 | { |
| 1433 | tags: [ |
| 1434 | "provider:fly", |
| 1435 | "provider:fly:app", |
| 1436 | "provider:fly:machine", |
| 1437 | "provider:fly:service", |
| 1438 | "live", |
| 1439 | ], |
| 1440 | }, |
| 1441 | () => { |
| 1442 | let endpoint: string | undefined; |
| 1443 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 1444 | |
| 1445 | test.provider( |
| 1446 | "F02 S07 harness forwards real Fly responses, loses completed responses and cuts connections", |
| 1447 | (stack) => |
| 1448 | Effect.gen(function* () { |
| 1449 | yield* stack.destroy(); |
| 1450 | const app = yield* stack.deploy(Fly.App("Site")); |
| 1451 | const proxy = yield* transportProxy(); |
| 1452 | yield* Effect.sync(() => { |
| 1453 | endpoint = proxy.url; |
| 1454 | }); |
| 1455 | try { |
| 1456 | const request = machines.getApp({ app_name: app.appName }).pipe(Retry.none); |
| 1457 | const forwarded = yield* request; |
| 1458 | expect(forwarded.name).toBe(app.appName); |
| 1459 | expect( |
| 1460 | proxy.events.some((event) => event.stage === "forwarded" && event.status === 200), |
| 1461 | ).toBe(true); |
| 1462 | yield* Effect.sync(() => |
| 1463 | proxy.arm({ |
| 1464 | match: (event) => event.path.endsWith(`/apps/${app.appName}`), |
| 1465 | action: "drop-response", |
| 1466 | remaining: 1, |
| 1467 | }), |
| 1468 | ); |
| 1469 | const dropped = yield* request.pipe(Effect.result); |
| 1470 | // The HTTP runtime may transparently replay an idempotent GET after a socket reset. |
| 1471 | if (Result.isFailure(dropped)) expect(dropped.failure._tag).toBe("HttpClientError"); |
| 1472 | else expect(dropped.success.name).toBe(app.appName); |
| 1473 | expect( |
| 1474 | proxy.events.filter((event) => event.stage === "dropped" && event.status === 200), |
| 1475 | ).toHaveLength(1); |
| 1476 | yield* Effect.sync(() => |
| 1477 | proxy.arm({ match: () => true, action: "cut-request", remaining: Infinity }), |
| 1478 | ); |
| 1479 | const cut = yield* request.pipe(Effect.result); |
| 1480 | expect(Result.isFailure(cut)).toBe(true); |
| 1481 | const cutEvent = proxy.events.find((event) => event.stage === "cut")!; |
| 1482 | expect( |
| 1483 | proxy.events.some( |
| 1484 | (event) => event.sequence === cutEvent.sequence && event.stage === "completed", |
| 1485 | ), |
| 1486 | ).toBe(false); |
| 1487 | yield* Effect.sync(() => { |
| 1488 | proxy.clear(); |
| 1489 | proxy.arm({ match: () => true, action: "hold-response", remaining: 1 }); |
| 1490 | }); |
| 1491 | const child = yield* request.pipe(Effect.forkScoped); |
| 1492 | const held = yield* proxy.wait((event) => event.stage === "held"); |
| 1493 | expect(held.status).toBe(200); |
| 1494 | yield* Effect.sync(proxy.release); |
| 1495 | expect((yield* Fiber.join(child).pipe(Effect.timeout("10 seconds"))).name).toBe( |
| 1496 | app.appName, |
| 1497 | ); |
| 1498 | expect((yield* request).name).toBe(app.appName); |
| 1499 | } finally { |
| 1500 | yield* Effect.sync(() => { |
| 1501 | endpoint = undefined; |
| 1502 | proxy.clear(); |
| 1503 | proxy.release(); |
| 1504 | }); |
| 1505 | } |
| 1506 | yield* stack.destroy(); |
| 1507 | yield* assertAppGone(app.appName); |
| 1508 | }).pipe( |
| 1509 | Effect.scoped, |
| 1510 | Effect.ensuring( |
| 1511 | Effect.sync(() => { |
| 1512 | endpoint = undefined; |
| 1513 | }), |
| 1514 | ), |
| 1515 | ), |
| 1516 | { timeout: 180_000 }, |
| 1517 | ); |
| 1518 | }, |
| 1519 | ); |
| 1520 | }); |
| 1521 | describe.sequential("shard 2", () => { |
| 1522 | describe.sequential( |
| 1523 | "workers", |
| 1524 | { |
| 1525 | tags: [ |
| 1526 | "provider:fly", |
| 1527 | "provider:fly:app", |
| 1528 | "provider:fly:ipassignment", |
| 1529 | "provider:fly:machine", |
| 1530 | "provider:fly:redis", |
| 1531 | "provider:fly:secret", |
| 1532 | "provider:fly:service", |
| 1533 | "live", |
| 1534 | ], |
| 1535 | }, |
| 1536 | () => { |
| 1537 | const { test } = Test.make({ providers: Fly.providers() }); |
| 1538 | |
| 1539 | // Cold gateway/worker publication, readiness, replacement and census can exceed three minutes. |
| 1540 | const timeout = 480_000; |
| 1541 | |
| 1542 | describe.sequential("Fly durable worker acceptance", () => { |
| 1543 | for (const mode of ["stop-delay", "stop-fail", "stop-hang"]) { |
| 1544 | test.provider( |
| 1545 | `R05 live ${mode} preserves another worker and bounds managed shutdown`, |
| 1546 | (stack) => |
| 1547 | Effect.gen(function* () { |
| 1548 | yield* stack.destroy(); |
| 1549 | const scenario = yield* makeScenario(stack); |
| 1550 | yield* Effect.gen(function* () { |
| 1551 | const first = yield* scenario.deploy({ |
| 1552 | version: "one", |
| 1553 | timeout: "10 seconds", |
| 1554 | signal: "SIGTERM", |
| 1555 | runOnly: true, |
| 1556 | mode, |
| 1557 | workers: 2, |
| 1558 | }); |
| 1559 | const worker = yield* requireValue(first.worker, "worker output"); |
| 1560 | const machineId = worker.machineId; |
| 1561 | const observe = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) => |
| 1562 | Effect.gen(function* () { |
| 1563 | const started = yield* Clock.currentTimeMillis; |
| 1564 | yield* Effect.logInfo("R05 phase started", { mode, machineId, phase }); |
| 1565 | return yield* effect.pipe( |
| 1566 | Effect.onExit((exit) => |
| 1567 | Clock.currentTimeMillis.pipe( |
| 1568 | Effect.flatMap((finished) => |
| 1569 | Effect.logInfo("R05 phase finished", { |
| 1570 | mode, |
| 1571 | machineId, |
| 1572 | phase, |
| 1573 | success: Exit.isSuccess(exit), |
| 1574 | elapsedMs: finished - started, |
| 1575 | }), |
| 1576 | ), |
| 1577 | ), |
| 1578 | ), |
| 1579 | ); |
| 1580 | }); |
| 1581 | yield* observe( |
| 1582 | "both-workers-ready-ledger", |
| 1583 | scenario.wait( |
| 1584 | first.ledgerUrl, |
| 1585 | (ledger) => |
| 1586 | ledger.events.filter( |
| 1587 | (event) => |
| 1588 | event.machine === machineId && event.event === "worker-ready", |
| 1589 | ).length === 2, |
| 1590 | ), |
| 1591 | ); |
| 1592 | yield* observe( |
| 1593 | "stop-machine-request", |
| 1594 | machines.stopMachine({ |
| 1595 | app_name: first.workerApp.appName, |
| 1596 | machine_id: machineId, |
| 1597 | }), |
| 1598 | ); |
| 1599 | const stopped = yield* observe( |
| 1600 | "stopped-machine-observation", |
| 1601 | machines |
| 1602 | .getMachine({ app_name: first.workerApp.appName, machine_id: machineId }) |
| 1603 | .pipe( |
| 1604 | Effect.repeat({ |
| 1605 | until: (value) => value.state === "stopped", |
| 1606 | times: 8, |
| 1607 | schedule: Schedule.spaced("2 seconds"), |
| 1608 | }), |
| 1609 | ), |
| 1610 | ); |
| 1611 | expect(stopped.state).toBe("stopped"); |
| 1612 | const ledger = yield* observe( |
| 1613 | "post-stop-ledger-snapshot", |
| 1614 | scenario.snapshot(first.ledgerUrl), |
| 1615 | ); |
| 1616 | assertOrder(ledger.events, machineId, "stopped", "drained", "b"); |
| 1617 | assertOrder(ledger.events, machineId, "drained", "client-released", "b"); |
| 1618 | const stopStarted = yield* requireValue( |
| 1619 | ledger.events.find( |
| 1620 | (event) => event.machine === machineId && event.event === "stop-started", |
| 1621 | ), |
| 1622 | "worker stop-started event", |
| 1623 | ); |
| 1624 | const independentRelease = yield* requireValue( |
| 1625 | ledger.events.find( |
| 1626 | (event) => |
| 1627 | event.machine === machineId && |
| 1628 | event.worker === "b" && |
| 1629 | event.event === "client-released", |
| 1630 | ), |
| 1631 | "worker b client release event", |
| 1632 | ); |
| 1633 | expect(independentRelease.at - stopStarted.at).toBeGreaterThanOrEqual(0); |
| 1634 | expect(independentRelease.at - stopStarted.at).toBeLessThan(10_000); |
| 1635 | const selected = ledger.events.filter( |
| 1636 | (event) => event.machine === machineId && event.worker === "a", |
| 1637 | ); |
| 1638 | if (mode === "stop-delay") { |
| 1639 | const a = yield* requireValue( |
| 1640 | selected.find((event) => event.event === "stopped"), |
| 1641 | "worker a stopped event", |
| 1642 | ); |
| 1643 | const b = yield* requireValue( |
| 1644 | ledger.events.find( |
| 1645 | (event) => |
| 1646 | event.machine === machineId && |
| 1647 | event.worker === "b" && |
| 1648 | event.event === "stopped", |
| 1649 | ), |
| 1650 | "worker b stopped event", |
| 1651 | ); |
| 1652 | expect(a.at - b.at).toBeGreaterThan(700); |
| 1653 | assertOrder(ledger.events, machineId, "stopped", "drained", "a"); |
| 1654 | assertStopped(ledger.events, machineId); |
| 1655 | } else { |
| 1656 | expect( |
| 1657 | selected.some( |
| 1658 | (event) => event.event === "stopped" || event.event === "drained", |
| 1659 | ), |
| 1660 | ).toBe(false); |
| 1661 | expect( |
| 1662 | selected.some( |
| 1663 | (event) => |
| 1664 | event.event === (mode === "stop-fail" ? "stop-failed" : "stop-started"), |
| 1665 | ), |
| 1666 | ).toBe(true); |
| 1667 | } |
| 1668 | if (mode === "stop-hang") { |
| 1669 | expect( |
| 1670 | selected.some( |
| 1671 | (event) => |
| 1672 | event.event === "work-closed" || event.event === "client-released", |
| 1673 | ), |
| 1674 | ).toBe(false); |
| 1675 | } |
| 1676 | expect( |
| 1677 | ledger.events.some( |
| 1678 | (event) => event.machine === machineId && event.event === "shared-closed", |
| 1679 | ), |
| 1680 | ).toBe(mode !== "stop-hang"); |
| 1681 | const exit = stopped.events?.find((event) => event.type === "exit")?.request as |
| 1682 | | { exit_event?: { exit_code?: number } } |
| 1683 | | undefined; |
| 1684 | yield* Effect.logInfo("Observed managed worker exit", { |
| 1685 | mode, |
| 1686 | exitCode: exit?.exit_event?.exit_code, |
| 1687 | }); |
| 1688 | expect(exit?.exit_event?.exit_code).toBe(mode === "stop-delay" ? 0 : 1); |
| 1689 | }).pipe(Effect.ensuring(scenario.cleanup)); |
| 1690 | }), |
| 1691 | { timeout }, |
| 1692 | ); |
| 1693 | } |
| 1694 | |
| 1695 | test.provider( |
| 1696 | "R07 authenticated durable Redis Streams gateway", |
| 1697 | (stack) => |
| 1698 | Effect.gen(function* () { |
| 1699 | yield* stack.destroy(); |
| 1700 | const scenario = yield* makeScenario(stack); |
| 1701 | yield* Effect.gen(function* () { |
| 1702 | const first = yield* scenario.deploy({ |
| 1703 | version: "one", |
| 1704 | timeout: "30 seconds", |
| 1705 | signal: "SIGTERM", |
| 1706 | runOnly: true, |
| 1707 | gatewayOnly: true, |
| 1708 | }); |
| 1709 | expect((yield* HttpClient.post(first.ledgerUrl)).status).toBe(401); |
| 1710 | const added = yield* scenario |
| 1711 | .call(first.ledgerUrl, "enqueue", ["durable-probe", "quick"]) |
| 1712 | .pipe(Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") })); |
| 1713 | expect(typeof added).toBe("string"); |
| 1714 | expect( |
| 1715 | yield* scenario.call(first.ledgerUrl, "enqueue", ["durable-probe", "quick"]), |
| 1716 | ).toBe(0); |
| 1717 | const claimed = JSON.parse( |
| 1718 | String(yield* scenario.call(first.ledgerUrl, "claim", ["probe-client", "one"])), |
| 1719 | ) as { id: string; job: string }; |
| 1720 | expect(claimed.job).toBe("durable-probe"); |
| 1721 | expect( |
| 1722 | yield* scenario.call(first.ledgerUrl, "finish", [ |
| 1723 | claimed.id, |
| 1724 | claimed.job, |
| 1725 | "probe-client", |
| 1726 | ]), |
| 1727 | ).toBe(1); |
| 1728 | const ledger = yield* scenario.settle(first.ledgerUrl); |
| 1729 | expect(ledger.pending).toBe(0); |
| 1730 | expect(ledger.results).toContain("durable-probe"); |
| 1731 | }).pipe(Effect.ensuring(scenario.cleanup)); |
| 1732 | }), |
| 1733 | { timeout }, |
| 1734 | ); |
| 1735 | |
| 1736 | for (const raw of [false, true]) { |
| 1737 | test.provider( |
| 1738 | `R07 ${raw ? "raw" : "managed"} private worker checkpoints and reclaims through replacement with producer overlap`, |
| 1739 | (stack) => |
| 1740 | Effect.gen(function* () { |
| 1741 | yield* stack.destroy(); |
| 1742 | const scenario = yield* makeScenario(stack); |
| 1743 | yield* Effect.gen(function* () { |
| 1744 | const options = { |
| 1745 | raw, |
| 1746 | runOnly: true, |
| 1747 | timeout: "30 seconds" as const, |
| 1748 | signal: "SIGTERM" as const, |
| 1749 | mode: "checkpoint", |
| 1750 | }; |
| 1751 | const first = yield* scenario.deploy({ ...options, version: "one" }); |
| 1752 | const oldWorker = yield* requireValue(first.worker, "initial worker output"); |
| 1753 | const oldId = oldWorker.machineId; |
| 1754 | yield* scenario.call(first.ledgerUrl, "enqueue", [ |
| 1755 | "checkpoint-job", |
| 1756 | "checkpoint", |
| 1757 | ]); |
| 1758 | yield* scenario.wait(first.ledgerUrl, (ledger) => |
| 1759 | ledger.events.some( |
| 1760 | (event) => |
| 1761 | event.machine === oldId && |
| 1762 | event.event === "claimed" && |
| 1763 | event.job === "checkpoint-job", |
| 1764 | ), |
| 1765 | ); |
| 1766 | const second = yield* scenario.deploy({ |
| 1767 | ...options, |
| 1768 | version: "two", |
| 1769 | mode: "drain", |
| 1770 | signal: "SIGINT", |
| 1771 | }); |
| 1772 | const newWorker = yield* requireValue( |
| 1773 | second.worker, |
| 1774 | "replacement worker output", |
| 1775 | ); |
| 1776 | const newId = newWorker.machineId; |
| 1777 | yield* assertReplacement(first.workerApp.appName, oldId, newId); |
| 1778 | const ledger = yield* scenario.settle(first.ledgerUrl); |
| 1779 | expect(ledger.results).toContain("checkpoint-job"); |
| 1780 | expect(ledger.checkpoints).toContain("saved-step-1"); |
| 1781 | expect( |
| 1782 | ledger.events.some( |
| 1783 | (event) => |
| 1784 | event.event === "reclaimed" && |
| 1785 | event.machine === newId && |
| 1786 | event.job === "checkpoint-job", |
| 1787 | ), |
| 1788 | ).toBe(true); |
| 1789 | const ack = ledger.events.filter( |
| 1790 | (event) => event.event === "acked" && event.job === "checkpoint-job", |
| 1791 | ); |
| 1792 | expect(ack).toHaveLength(1); |
| 1793 | expect(ack[0]?.ack).toBe(1); |
| 1794 | expect(ack[0]?.fresh).toBe(1); |
| 1795 | assertStopped(ledger.events, oldId); |
| 1796 | assertOrder(ledger.events, oldId, "stopped", "checkpoint"); |
| 1797 | assertOrder(ledger.events, oldId, "checkpoint", "client-released"); |
| 1798 | assertOrder(ledger.events, oldId, "client-released", "shared-closed"); |
| 1799 | const oldSlots = new Set( |
| 1800 | ledger.events |
| 1801 | .filter((event) => event.machine === oldId && event.event === "producer") |
| 1802 | .map((event) => event.job), |
| 1803 | ); |
| 1804 | const overlaps = ledger.events.filter( |
| 1805 | (event) => |
| 1806 | event.machine === newId && |
| 1807 | event.event === "producer" && |
| 1808 | oldSlots.has(event.job), |
| 1809 | ); |
| 1810 | expect(overlaps.length).toBeGreaterThan(0); |
| 1811 | for (const event of overlaps) { |
| 1812 | expect( |
| 1813 | ledger.events.filter( |
| 1814 | (row) => |
| 1815 | row.event === "producer" && row.job === event.job && row.fresh === 1, |
| 1816 | ), |
| 1817 | ).toHaveLength(1); |
| 1818 | } |
| 1819 | }).pipe(Effect.ensuring(scenario.cleanup)); |
| 1820 | }), |
| 1821 | { timeout }, |
| 1822 | ); |
| 1823 | } |
| 1824 | }); |
| 1825 | }, |
| 1826 | ); |
| 1827 | |
| 1828 | describe.sequential( |
| 1829 | "concurrency", |
| 1830 | { |
| 1831 | tags: [ |
| 1832 | "provider:fly", |
| 1833 | "provider:fly:app", |
| 1834 | "provider:fly:machine", |
| 1835 | "provider:fly:service", |
| 1836 | "live", |
| 1837 | ], |
| 1838 | }, |
| 1839 | () => { |
| 1840 | const file = "test/Fly/BlueGreen.test.ts"; |
| 1841 | const { test } = Test.make({ providers: Fly.providers() }); |
| 1842 | |
| 1843 | test.provider( |
| 1844 | "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness", |
| 1845 | (stack) => |
| 1846 | Effect.gen(function* () { |
| 1847 | yield* stack.destroy(); |
| 1848 | const initial = yield* deployWorker(stack, "one"); |
| 1849 | const proxy = yield* transportProxy(); |
| 1850 | yield* Effect.sync(() => |
| 1851 | proxy.arm({ |
| 1852 | match: (event) => event.method === "GET" && event.path.endsWith("/machines"), |
| 1853 | action: "hold-response", |
| 1854 | remaining: 1, |
| 1855 | }), |
| 1856 | ); |
| 1857 | yield* Effect.gen(function* () { |
| 1858 | const staleActor = yield* engineActor( |
| 1859 | stack, |
| 1860 | "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness", |
| 1861 | file, |
| 1862 | proxy.url, |
| 1863 | ); |
| 1864 | const freshActor = yield* engineActor( |
| 1865 | stack, |
| 1866 | "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness", |
| 1867 | file, |
| 1868 | ); |
| 1869 | const delayed = yield* deployWorker(staleActor, "two").pipe( |
| 1870 | Effect.scoped, |
| 1871 | Effect.result, |
| 1872 | Effect.forkScoped, |
| 1873 | ); |
| 1874 | yield* proxy.wait((event) => event.stage === "held" && event.status === 200); |
| 1875 | const newer = yield* deployWorker(freshActor, "three"); |
| 1876 | expect(newer.machineId).not.toBe(initial.machineId); |
| 1877 | expect( |
| 1878 | proxy.events.some((event) => event.stage === "request" && event.method !== "GET"), |
| 1879 | ).toBe(false); |
| 1880 | yield* Effect.sync(proxy.release); |
| 1881 | const result = yield* Fiber.join(delayed).pipe(Effect.timeout("180 seconds")); |
| 1882 | if (Result.isSuccess(result)) { |
| 1883 | // A later valid reconcile may win; native leases do not fence LocalState with a global epoch. |
| 1884 | yield* assertCommitted(initial.appName, result.success.machineIds); |
| 1885 | for (const id of newer.machineIds) { |
| 1886 | const retire = proxy.events.findIndex( |
| 1887 | (event) => |
| 1888 | event.stage === "request" && |
| 1889 | event.machineId === id && |
| 1890 | (event.path.endsWith("/stop") || |
| 1891 | event.path.endsWith("/cordon") || |
| 1892 | (event.method === "DELETE" && !event.path.endsWith("/lease"))), |
| 1893 | ); |
| 1894 | expect(retire).toBeGreaterThan(0); |
| 1895 | for (const candidate of result.success.machineIds) { |
| 1896 | expect( |
| 1897 | proxy.events |
| 1898 | .slice(0, retire) |
| 1899 | .some( |
| 1900 | (event) => |
| 1901 | event.stage === "completed" && |
| 1902 | event.machineId === candidate && |
| 1903 | event.method === "GET" && |
| 1904 | event.state === "started" && |
| 1905 | event.checks?.some( |
| 1906 | (check) => check.name === "ready" && check.status === "passing", |
| 1907 | ), |
| 1908 | ), |
| 1909 | ).toBe(true); |
| 1910 | } |
| 1911 | } |
| 1912 | } else { |
| 1913 | expect([ |
| 1914 | "Fly.DeploymentRecoveryAmbiguous", |
| 1915 | "Fly.ReplicaOwnershipChanged", |
| 1916 | "Fly.MachineLeaseBusy", |
| 1917 | "Fly.MachineLeaseLost", |
| 1918 | "NotFound", |
| 1919 | ]).toContain(result.failure._tag); |
| 1920 | expect( |
| 1921 | (yield* census(initial.appName)).some( |
| 1922 | (machine) => machine.id === newer.machineId, |
| 1923 | ), |
| 1924 | ).toBe(true); |
| 1925 | yield* Effect.logInfo("Stale snapshot refused without state-fencing guarantee", { |
| 1926 | outcome: result.failure._tag, |
| 1927 | }); |
| 1928 | } |
| 1929 | const resumed = yield* engineActor( |
| 1930 | stack, |
| 1931 | "F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness", |
| 1932 | file, |
| 1933 | ); |
| 1934 | const settled = yield* deployWorker(resumed, "four"); |
| 1935 | yield* assertCommitted(initial.appName, settled.machineIds); |
| 1936 | }).pipe( |
| 1937 | Effect.ensuring( |
| 1938 | Effect.sync(() => { |
| 1939 | proxy.clear(); |
| 1940 | proxy.release(); |
| 1941 | }), |
| 1942 | ), |
| 1943 | Effect.scoped, |
| 1944 | ); |
| 1945 | yield* stack.destroy(); |
| 1946 | yield* assertAppGone(initial.appName); |
| 1947 | }).pipe(Effect.scoped), |
| 1948 | { timeout: 600_000 }, |
| 1949 | ); |
| 1950 | |
| 1951 | describe.sequential("independent engine contexts", () => { |
| 1952 | for (const competitor of ["bluegreen", "rolling", "destroy"] as const) { |
| 1953 | test.provider( |
| 1954 | `F11 shared-old lease excludes ${competitor} while a candidate response is held`, |
| 1955 | (stack) => |
| 1956 | Effect.gen(function* () { |
| 1957 | yield* stack.destroy(); |
| 1958 | const initial = yield* deployWorker(stack, "one"); |
| 1959 | const holderProxy = yield* transportProxy(); |
| 1960 | const contenderProxy = yield* transportProxy(); |
| 1961 | yield* Effect.sync(() => |
| 1962 | holderProxy.arm({ |
| 1963 | match: (event) => event.method === "POST" && event.path.endsWith("/machines"), |
| 1964 | action: "hold-response", |
| 1965 | remaining: 1, |
| 1966 | }), |
| 1967 | ); |
| 1968 | yield* Effect.gen(function* () { |
| 1969 | const holderActor = yield* engineActor( |
| 1970 | stack, |
| 1971 | `F11 shared-old lease excludes ${competitor} while a candidate response is held`, |
| 1972 | file, |
| 1973 | holderProxy.url, |
| 1974 | ); |
| 1975 | const contenderActor = yield* engineActor( |
| 1976 | stack, |
| 1977 | `F11 shared-old lease excludes ${competitor} while a candidate response is held`, |
| 1978 | file, |
| 1979 | contenderProxy.url, |
| 1980 | ); |
| 1981 | expect(contenderActor.state).not.toBe(holderActor.state); |
| 1982 | const holder = yield* deployWorker(holderActor, "two").pipe( |
| 1983 | Effect.scoped, |
| 1984 | Effect.forkScoped, |
| 1985 | ); |
| 1986 | const held = yield* holderProxy.wait( |
| 1987 | (event) => event.stage === "held" && event.status! < 300, |
| 1988 | ); |
| 1989 | const operation = |
| 1990 | competitor === "destroy" |
| 1991 | ? contenderActor.destroy() |
| 1992 | : deployWorker(contenderActor, "three", { |
| 1993 | deploy: { strategy: competitor, healthTimeout: "30 seconds" }, |
| 1994 | }).pipe(Effect.asVoid); |
| 1995 | const result = yield* operation.pipe( |
| 1996 | Effect.scoped, |
| 1997 | Effect.timeout("75 seconds"), |
| 1998 | Effect.result, |
| 1999 | ); |
| 2000 | yield* Effect.sync(holderProxy.release); |
| 2001 | expect(Result.isFailure(result)).toBe(true); |
| 2002 | if (Result.isFailure(result)) |
| 2003 | expect(result.failure).not.toMatchObject({ _tag: "TimeoutError" }); |
| 2004 | expect( |
| 2005 | contenderProxy.events.some( |
| 2006 | (event) => |
| 2007 | event.stage === "completed" && |
| 2008 | event.status === 409 && |
| 2009 | event.path.endsWith("/lease") && |
| 2010 | event.machineId === initial.machineId, |
| 2011 | ), |
| 2012 | ).toBe(true); |
| 2013 | expect( |
| 2014 | contenderProxy.events.some( |
| 2015 | (event) => |
| 2016 | event.stage === "completed" && |
| 2017 | event.status! < 300 && |
| 2018 | event.method !== "GET" && |
| 2019 | !event.path.endsWith("/lease"), |
| 2020 | ), |
| 2021 | ).toBe(false); |
| 2022 | const next = yield* Fiber.join(holder).pipe(Effect.timeout("90 seconds")); |
| 2023 | expect(next.machineIds).toEqual([held.machineId]); |
| 2024 | expect((yield* census(initial.appName)).map((machine) => machine.id)).toEqual( |
| 2025 | next.machineIds, |
| 2026 | ); |
| 2027 | }).pipe( |
| 2028 | Effect.ensuring( |
| 2029 | Effect.sync(() => { |
| 2030 | holderProxy.clear(); |
| 2031 | holderProxy.release(); |
| 2032 | }), |
| 2033 | ), |
| 2034 | Effect.scoped, |
| 2035 | ); |
| 2036 | yield* stack.destroy(); |
| 2037 | yield* assertAppGone(initial.appName); |
| 2038 | }).pipe(Effect.scoped), |
| 2039 | { timeout: 300_000 }, |
| 2040 | ); |
| 2041 | } |
| 2042 | |
| 2043 | test.provider( |
| 2044 | "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock", |
| 2045 | (stack) => |
| 2046 | Effect.gen(function* () { |
| 2047 | yield* stack.destroy(); |
| 2048 | const app = yield* stack.deploy(Fly.App("Site")); |
| 2049 | const firstProxy = yield* transportProxy(); |
| 2050 | const secondProxy = yield* transportProxy(); |
| 2051 | yield* Effect.sync(() => |
| 2052 | firstProxy.arm({ |
| 2053 | match: (event) => event.method === "POST" && event.path.endsWith("/machines"), |
| 2054 | action: "hold-response", |
| 2055 | remaining: 1, |
| 2056 | }), |
| 2057 | ); |
| 2058 | yield* Effect.gen(function* () { |
| 2059 | const firstActor = yield* engineActor( |
| 2060 | stack, |
| 2061 | "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock", |
| 2062 | file, |
| 2063 | firstProxy.url, |
| 2064 | ); |
| 2065 | const secondActor = yield* engineActor( |
| 2066 | stack, |
| 2067 | "S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock", |
| 2068 | file, |
| 2069 | secondProxy.url, |
| 2070 | ); |
| 2071 | const first = yield* deployWorker(firstActor, "one").pipe( |
| 2072 | Effect.scoped, |
| 2073 | Effect.result, |
| 2074 | Effect.forkScoped, |
| 2075 | ); |
| 2076 | const created = yield* firstProxy.wait( |
| 2077 | (event) => event.stage === "held" && event.status! < 300, |
| 2078 | ); |
| 2079 | const original = (yield* census(app.appName)).find( |
| 2080 | (machine) => machine.id === created.machineId, |
| 2081 | ); |
| 2082 | const owner = original?.config?.metadata; |
| 2083 | expect(owner?.["alchemy.fqn"]).toBeTruthy(); |
| 2084 | expect(owner?.["alchemy.instance"]).toBeTruthy(); |
| 2085 | expect( |
| 2086 | firstProxy.events.some( |
| 2087 | (event) => |
| 2088 | event.stage === "completed" && |
| 2089 | event.method === "POST" && |
| 2090 | event.path.endsWith("/lease") && |
| 2091 | event.status! < 300, |
| 2092 | ), |
| 2093 | ).toBe(false); |
| 2094 | const second = yield* deployWorker(secondActor, "two").pipe( |
| 2095 | Effect.scoped, |
| 2096 | Effect.timeout("90 seconds"), |
| 2097 | Effect.result, |
| 2098 | ); |
| 2099 | const during = yield* census(app.appName); |
| 2100 | yield* Effect.sync(firstProxy.release); |
| 2101 | const firstResult = yield* Fiber.join(first).pipe(Effect.timeout("90 seconds")); |
| 2102 | yield* Effect.logInfo("First-deploy results: no global exclusion promised", { |
| 2103 | first: Result.isFailure(firstResult) |
| 2104 | ? firstResult.failure._tag |
| 2105 | : firstResult._tag, |
| 2106 | second: Result.isFailure(second) ? second.failure._tag : second._tag, |
| 2107 | }); |
| 2108 | const successfulIds: string[] = []; |
| 2109 | for (const result of [firstResult, second]) { |
| 2110 | if (Result.isSuccess(result)) { |
| 2111 | expect(result.success.machineIds.length).toBe(1); |
| 2112 | successfulIds.push(...result.success.machineIds); |
| 2113 | } else { |
| 2114 | expect([ |
| 2115 | "Fly.DeploymentRecoveryAmbiguous", |
| 2116 | "Fly.ReplicaOwnershipChanged", |
| 2117 | "Fly.MachineLeaseBusy", |
| 2118 | "Fly.MachineLeaseLost", |
| 2119 | "Fly.MachineNotCreated", |
| 2120 | "NotFound", |
| 2121 | ]).toContain(result.failure._tag); |
| 2122 | } |
| 2123 | } |
| 2124 | expect(successfulIds.length).toBeGreaterThan(0); |
| 2125 | // A held create response conveys no lease; a leased, ready successor may retire it. |
| 2126 | if (Result.isSuccess(second)) { |
| 2127 | const committed = yield* assertCommitted( |
| 2128 | app.appName, |
| 2129 | second.success.machineIds, |
| 2130 | ); |
| 2131 | for (const machine of committed) { |
| 2132 | yield* waitHealthy(app.appName, machine, 30_000, machine.config); |
| 2133 | } |
| 2134 | expect(during.map((machine) => machine.id).sort()).toEqual( |
| 2135 | [...second.success.machineIds].sort(), |
| 2136 | ); |
| 2137 | expect( |
| 2138 | during.every( |
| 2139 | (machine) => |
| 2140 | machine.state === "started" && |
| 2141 | machine.cordoned === false && |
| 2142 | machine.config?.metadata?.["alchemy.phase"] === "active", |
| 2143 | ), |
| 2144 | ).toBe(true); |
| 2145 | } else { |
| 2146 | expect(during.some((machine) => machine.id === created.machineId)).toBe(true); |
| 2147 | } |
| 2148 | const createdIds = new Set( |
| 2149 | [firstProxy, secondProxy].flatMap((proxy) => |
| 2150 | proxy.events |
| 2151 | .filter( |
| 2152 | (event) => |
| 2153 | event.stage === "completed" && |
| 2154 | event.method === "POST" && |
| 2155 | event.path.endsWith("/machines") && |
| 2156 | event.status! < 300, |
| 2157 | ) |
| 2158 | .map((event) => event.machineId), |
| 2159 | ), |
| 2160 | ); |
| 2161 | for (const proxy of [firstProxy, secondProxy]) { |
| 2162 | for (const [index, event] of proxy.events.entries()) { |
| 2163 | if ( |
| 2164 | event.stage !== "request" || |
| 2165 | !( |
| 2166 | event.path.endsWith("/stop") || |
| 2167 | event.path.endsWith("/cordon") || |
| 2168 | event.phase === "retiring" || |
| 2169 | (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path)) |
| 2170 | ) |
| 2171 | ) |
| 2172 | continue; |
| 2173 | expect(event.machineId).toBeDefined(); |
| 2174 | expect(createdIds.has(event.machineId)).toBe(true); |
| 2175 | const before = proxy.events.slice(0, index); |
| 2176 | const lease = before.findIndex( |
| 2177 | (prior) => |
| 2178 | prior.stage === "completed" && |
| 2179 | prior.machineId === event.machineId && |
| 2180 | prior.method === "POST" && |
| 2181 | prior.path.endsWith("/lease") && |
| 2182 | prior.status! < 300, |
| 2183 | ); |
| 2184 | expect(lease).toBeGreaterThan(-1); |
| 2185 | expect( |
| 2186 | before |
| 2187 | .slice(lease + 1) |
| 2188 | .some( |
| 2189 | (prior) => |
| 2190 | prior.stage === "completed" && |
| 2191 | prior.machineId === event.machineId && |
| 2192 | prior.method === "GET" && |
| 2193 | /\/machines\/[^/]+$/.test(prior.path) && |
| 2194 | prior.status === 200, |
| 2195 | ), |
| 2196 | ).toBe(true); |
| 2197 | expect( |
| 2198 | before.some( |
| 2199 | (ready, readyIndex) => |
| 2200 | ready.stage === "completed" && |
| 2201 | ready.method === "GET" && |
| 2202 | ready.status === 200 && |
| 2203 | ready.machineId !== event.machineId && |
| 2204 | createdIds.has(ready.machineId) && |
| 2205 | ready.state === "started" && |
| 2206 | ready.cordoned === false && |
| 2207 | ready.checks?.some( |
| 2208 | (check) => check.name === "ready" && check.status === "passing", |
| 2209 | ) && |
| 2210 | before |
| 2211 | .slice(readyIndex + 1) |
| 2212 | .some( |
| 2213 | (active) => |
| 2214 | active.stage === "completed" && |
| 2215 | active.machineId === ready.machineId && |
| 2216 | active.method === "PATCH" && |
| 2217 | active.phase === "active" && |
| 2218 | active.status! < 300, |
| 2219 | ), |
| 2220 | ), |
| 2221 | ).toBe(true); |
| 2222 | } |
| 2223 | } |
| 2224 | const live = yield* census(app.appName); |
| 2225 | expect(live.length).toBe(1); |
| 2226 | expect( |
| 2227 | live.every( |
| 2228 | (machine) => |
| 2229 | successfulIds.includes(machine.id!) && |
| 2230 | machine.state === "started" && |
| 2231 | machine.cordoned === false && |
| 2232 | machine.checks?.some( |
| 2233 | (check) => check.name === "ready" && check.status === "passing", |
| 2234 | ) && |
| 2235 | machine.config?.metadata?.["alchemy.fqn"] === owner?.["alchemy.fqn"] && |
| 2236 | machine.config?.metadata?.["alchemy.instance"] === |
| 2237 | owner?.["alchemy.instance"], |
| 2238 | ), |
| 2239 | ).toBe(true); |
| 2240 | yield* assertCommitted( |
| 2241 | app.appName, |
| 2242 | live.map((machine) => machine.id!), |
| 2243 | ); |
| 2244 | }).pipe( |
| 2245 | Effect.ensuring( |
| 2246 | Effect.sync(() => { |
| 2247 | firstProxy.clear(); |
| 2248 | firstProxy.release(); |
| 2249 | }), |
| 2250 | ), |
| 2251 | Effect.scoped, |
| 2252 | ); |
| 2253 | yield* stack.destroy(); |
| 2254 | yield* assertAppGone(app.appName); |
| 2255 | }).pipe(Effect.scoped), |
| 2256 | { timeout: 300_000 }, |
| 2257 | ); |
| 2258 | }); |
| 2259 | }, |
| 2260 | ); |
| 2261 | |
| 2262 | describe.sequential( |
| 2263 | "commit recovery", |
| 2264 | { |
| 2265 | tags: [ |
| 2266 | "provider:fly", |
| 2267 | "provider:fly:app", |
| 2268 | "provider:fly:ipassignment", |
| 2269 | "provider:fly:machine", |
| 2270 | "provider:fly:secret", |
| 2271 | "provider:fly:service", |
| 2272 | "live", |
| 2273 | ], |
| 2274 | }, |
| 2275 | () => { |
| 2276 | const file = "test/Fly/BlueGreen.test.ts"; |
| 2277 | const { test } = Test.make({ providers: Fly.providers() }); |
| 2278 | |
| 2279 | for (const interrupted of [false, true]) { |
| 2280 | test.provider( |
| 2281 | `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`, |
| 2282 | (stack) => |
| 2283 | Effect.gen(function* () { |
| 2284 | yield* stack.destroy(); |
| 2285 | const readiness = yield* makeReadinessControl(); |
| 2286 | const site = yield* readiness.deployApp(stack); |
| 2287 | const proxy = yield* transportProxy(); |
| 2288 | yield* Effect.sync(() => |
| 2289 | proxy.arm({ |
| 2290 | match: (event) => |
| 2291 | event.path.endsWith("/metadata") && event.phase === "validating", |
| 2292 | action: "hold-response", |
| 2293 | remaining: 1, |
| 2294 | }), |
| 2295 | ); |
| 2296 | const actor = yield* engineActor( |
| 2297 | stack, |
| 2298 | `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`, |
| 2299 | file, |
| 2300 | proxy.url, |
| 2301 | ); |
| 2302 | const attempt = yield* readiness |
| 2303 | .deployWorker(actor, "one") |
| 2304 | .pipe(Effect.scoped, Effect.result, Effect.forkScoped); |
| 2305 | const barrier = yield* proxy |
| 2306 | .wait( |
| 2307 | (event) => |
| 2308 | event.stage === "held" && |
| 2309 | event.phase === "validating" && |
| 2310 | event.status! >= 200 && |
| 2311 | event.status! < 300, |
| 2312 | ) |
| 2313 | .pipe( |
| 2314 | Effect.raceFirst( |
| 2315 | Effect.gen(function* () { |
| 2316 | const result = yield* Fiber.join(attempt); |
| 2317 | if (Result.isFailure(result)) return yield* Effect.fail(result.failure); |
| 2318 | return yield* Effect.fail( |
| 2319 | new Error("Deployment completed without reaching the validating barrier"), |
| 2320 | ); |
| 2321 | }), |
| 2322 | ), |
| 2323 | ); |
| 2324 | const machineId = barrier.machineId!; |
| 2325 | expect(machineId).toBeDefined(); |
| 2326 | |
| 2327 | const firstRouting = proxy.events.findIndex( |
| 2328 | (event) => event.stage === "request" && event.path.endsWith("/uncordon"), |
| 2329 | ); |
| 2330 | expect(firstRouting).toBeGreaterThan(0); |
| 2331 | const preparation = proxy.events.slice(0, firstRouting); |
| 2332 | const promoting = preparation.findLastIndex( |
| 2333 | (event) => |
| 2334 | event.stage === "completed" && |
| 2335 | event.phase === "promoting" && |
| 2336 | event.path.endsWith("/metadata") && |
| 2337 | event.status! < 300, |
| 2338 | ); |
| 2339 | const ready = preparation.findLastIndex( |
| 2340 | (event) => |
| 2341 | event.stage === "completed" && |
| 2342 | event.method === "GET" && |
| 2343 | event.path.endsWith(`/machines/${machineId}`) && |
| 2344 | event.state === "started" && |
| 2345 | event.cordoned === true && |
| 2346 | event.checks?.find((check) => check.name === "ready")?.status === "passing", |
| 2347 | ); |
| 2348 | expect(promoting).toBeGreaterThanOrEqual(0); |
| 2349 | expect(ready).toBeGreaterThan(promoting); |
| 2350 | |
| 2351 | yield* readiness.turnOff(site.appName, machineId); |
| 2352 | if (interrupted) { |
| 2353 | const interruption = yield* Fiber.interrupt(attempt).pipe(Effect.forkScoped); |
| 2354 | yield* Effect.yieldNow; |
| 2355 | yield* Effect.sync(proxy.release); |
| 2356 | yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds")); |
| 2357 | expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true); |
| 2358 | } else { |
| 2359 | yield* Effect.sync(proxy.release); |
| 2360 | const result = yield* Fiber.join(attempt).pipe(Effect.timeout("180 seconds")); |
| 2361 | expect(Result.isFailure(result)).toBe(true); |
| 2362 | if (Result.isFailure(result)) |
| 2363 | expect(result.failure._tag).toBe("Fly.ReplicaChecksNotPassing"); |
| 2364 | } |
| 2365 | yield* Effect.sync(proxy.clear); |
| 2366 | const pending = yield* census(site.appName); |
| 2367 | expect(pending.map((machine) => machine.id)).toEqual([machineId]); |
| 2368 | expect(pending[0]!.cordoned).toBe(false); |
| 2369 | const metadata = pending[0]!.config!.metadata!; |
| 2370 | expect(metadata["alchemy.phase"]).toBe("validating"); |
| 2371 | expect(metadata["alchemy.checked-instance"]).toBeUndefined(); |
| 2372 | const read = yield* observeReplicaSet({ |
| 2373 | appName: site.appName, |
| 2374 | id: "Worker", |
| 2375 | type: "Fly.Machine", |
| 2376 | fqn: metadata["alchemy.fqn"]!, |
| 2377 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 2378 | machineIds: [machineId], |
| 2379 | }); |
| 2380 | expect(read?.rolloutPending).toBe(true); |
| 2381 | expect(read?.machineIds).toEqual([]); |
| 2382 | |
| 2383 | const retryActor = yield* engineActor( |
| 2384 | stack, |
| 2385 | `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`, |
| 2386 | file, |
| 2387 | proxy.url, |
| 2388 | ); |
| 2389 | const stillBroken = yield* readiness |
| 2390 | .deployWorker(retryActor, "one") |
| 2391 | .pipe(Effect.scoped, Effect.result); |
| 2392 | expect(Result.isFailure(stillBroken)).toBe(true); |
| 2393 | if (Result.isFailure(stillBroken)) |
| 2394 | expect(stillBroken.failure._tag).toBe("Fly.ReplicaChecksNotPassing"); |
| 2395 | expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual([ |
| 2396 | machineId, |
| 2397 | ]); |
| 2398 | expect( |
| 2399 | proxy.events.some( |
| 2400 | (event) => event.phase === "active" && event.path.endsWith("/metadata"), |
| 2401 | ), |
| 2402 | ).toBe(false); |
| 2403 | |
| 2404 | yield* repairReadiness(site.appName, machineId); |
| 2405 | const recoveryActor = yield* engineActor( |
| 2406 | stack, |
| 2407 | `FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`, |
| 2408 | file, |
| 2409 | proxy.url, |
| 2410 | ); |
| 2411 | const recovered = yield* readiness |
| 2412 | .deployWorker(recoveryActor, "one") |
| 2413 | .pipe(Effect.scoped); |
| 2414 | expect(recovered.machineIds).toEqual([machineId]); |
| 2415 | yield* assertCommitted(site.appName, recovered.machineIds); |
| 2416 | const committed = yield* observeReplicaSet({ |
| 2417 | appName: site.appName, |
| 2418 | id: "Worker", |
| 2419 | type: "Fly.Machine", |
| 2420 | fqn: metadata["alchemy.fqn"]!, |
| 2421 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 2422 | machineIds: [machineId], |
| 2423 | }); |
| 2424 | expect(committed?.rolloutPending).toBe(false); |
| 2425 | expect(committed?.machineIds).toEqual([machineId]); |
| 2426 | yield* stack.destroy(); |
| 2427 | yield* assertAppGone(site.appName); |
| 2428 | }).pipe(Effect.scoped), |
| 2429 | { timeout: 600_000 }, |
| 2430 | ); |
| 2431 | } |
| 2432 | }, |
| 2433 | ); |
| 2434 | |
| 2435 | describe.sequential( |
| 2436 | "health", |
| 2437 | { |
| 2438 | tags: [ |
| 2439 | "provider:fly", |
| 2440 | "provider:fly:app", |
| 2441 | "provider:fly:machine", |
| 2442 | "provider:fly:service", |
| 2443 | "live", |
| 2444 | ], |
| 2445 | }, |
| 2446 | () => { |
| 2447 | const { test } = Test.make({ providers: Fly.providers() }); |
| 2448 | |
| 2449 | for (const kind of ["named", "multiple published services"] as const) { |
| 2450 | test.provider( |
| 2451 | `S03 every ${kind} check must pass before blue retirement`, |
| 2452 | (stack) => |
| 2453 | Effect.gen(function* () { |
| 2454 | yield* stack.destroy(); |
| 2455 | const initial = yield* deployWorker(stack, "one", { |
| 2456 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 2457 | }); |
| 2458 | const extra = { ...checks.ready, path: "/does-not-exist" }; |
| 2459 | const failed = yield* deployWorker(stack, "two", { |
| 2460 | checks: kind === "named" ? { ready: checks.ready, dependency: extra } : checks, |
| 2461 | services: |
| 2462 | kind === "named" |
| 2463 | ? undefined |
| 2464 | : [ |
| 2465 | { |
| 2466 | protocol: "tcp", |
| 2467 | internalPort: 80, |
| 2468 | ports: [{ port: 80, handlers: ["http"] }], |
| 2469 | checks: [checks.ready], |
| 2470 | }, |
| 2471 | { |
| 2472 | protocol: "tcp", |
| 2473 | internalPort: 80, |
| 2474 | ports: [{ port: 443, handlers: ["tls", "http"] }], |
| 2475 | checks: [extra], |
| 2476 | }, |
| 2477 | ], |
| 2478 | deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" }, |
| 2479 | }).pipe(Effect.result); |
| 2480 | expect(Result.isFailure(failed)).toBe(true); |
| 2481 | if (Result.isFailure(failed)) |
| 2482 | expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" }); |
| 2483 | const live = yield* census(initial.appName); |
| 2484 | expect(live.map((machine) => machine.id)).toEqual(initial.machineIds); |
| 2485 | expect(live[0]!.cordoned).toBe(false); |
| 2486 | const fixed = yield* deployWorker(stack, "two", { |
| 2487 | checks: { ready: checks.ready, dependency: checks.ready }, |
| 2488 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 2489 | }); |
| 2490 | const committed = yield* assertCommitted(initial.appName, fixed.machineIds); |
| 2491 | // Active commit metadata can reset reports after readiness validation. |
| 2492 | const healthy = yield* waitHealthy(initial.appName, committed[0]!, 30_000); |
| 2493 | expect(healthy.instance_id).toBe(committed[0]!.instance_id); |
| 2494 | expect( |
| 2495 | ["ready", "dependency"].every((name) => |
| 2496 | healthy.checks?.some( |
| 2497 | (check) => check.name === name && check.status === "passing", |
| 2498 | ), |
| 2499 | ), |
| 2500 | ).toBe(true); |
| 2501 | yield* stack.destroy(); |
| 2502 | yield* assertAppGone(initial.appName); |
| 2503 | }), |
| 2504 | { timeout: 300_000 }, |
| 2505 | ); |
| 2506 | } |
| 2507 | }, |
| 2508 | ); |
| 2509 | |
| 2510 | describe.sequential( |
| 2511 | "required metadata writes", |
| 2512 | { |
| 2513 | tags: [ |
| 2514 | "provider:fly", |
| 2515 | "provider:fly:app", |
| 2516 | "provider:fly:machine", |
| 2517 | "provider:fly:service", |
| 2518 | "live", |
| 2519 | ], |
| 2520 | }, |
| 2521 | () => { |
| 2522 | let endpoint: string | undefined; |
| 2523 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 2524 | |
| 2525 | const props = { |
| 2526 | count: 2, |
| 2527 | deploy: { strategy: "bluegreen", healthTimeout: "90 seconds" }, |
| 2528 | } as const; |
| 2529 | |
| 2530 | test.provider( |
| 2531 | "F05 real required promotion-intent write fault forbids every uncordon and early old deletion", |
| 2532 | (stack) => |
| 2533 | Effect.gen(function* () { |
| 2534 | yield* stack.destroy(); |
| 2535 | const initial = yield* deployWorker(stack, "one", props); |
| 2536 | const proxy = yield* transportProxy(); |
| 2537 | try { |
| 2538 | yield* Effect.sync(() => { |
| 2539 | endpoint = proxy.url; |
| 2540 | proxy.arm({ |
| 2541 | match: (event) => |
| 2542 | event.method === "PATCH" && |
| 2543 | event.path.endsWith("/metadata") && |
| 2544 | event.phase === "promoting", |
| 2545 | action: "cut-request", |
| 2546 | remaining: Infinity, |
| 2547 | }); |
| 2548 | }); |
| 2549 | const result = yield* deployWorker(stack, "two", props).pipe(Effect.result); |
| 2550 | expect(Result.isFailure(result)).toBe(true); |
| 2551 | if (Result.isFailure(result)) |
| 2552 | expect(result.failure).toMatchObject({ _tag: "Fly.MachineMutationUncertain" }); |
| 2553 | const cut = proxy.events.filter( |
| 2554 | (event) => event.stage === "cut" && event.phase === "promoting", |
| 2555 | ); |
| 2556 | expect(cut.length).toBeGreaterThan(0); |
| 2557 | expect(new Set(cut.map((event) => event.machineId)).size).toBe(1); |
| 2558 | const live = yield* census(initial.appName); |
| 2559 | const old = live.filter((machine) => initial.machineIds.includes(machine.id!)); |
| 2560 | const candidates = live.filter( |
| 2561 | (machine) => !initial.machineIds.includes(machine.id!), |
| 2562 | ); |
| 2563 | expect(old.map((machine) => machine.id).sort()).toEqual( |
| 2564 | [...initial.machineIds].sort(), |
| 2565 | ); |
| 2566 | expect( |
| 2567 | old.every((machine) => machine.cordoned === false && machine.state === "started"), |
| 2568 | ).toBe(true); |
| 2569 | expect(candidates).toHaveLength(2); |
| 2570 | expect( |
| 2571 | candidates.every( |
| 2572 | (machine) => |
| 2573 | machine.cordoned === true && |
| 2574 | machine.config?.metadata?.["alchemy.phase"] === "candidate", |
| 2575 | ), |
| 2576 | ).toBe(true); |
| 2577 | expect(candidates.some((machine) => machine.id === cut[0]!.machineId)).toBe(true); |
| 2578 | expect(proxy.events.some((event) => event.path.endsWith("/uncordon"))).toBe(false); |
| 2579 | expect( |
| 2580 | proxy.events.some( |
| 2581 | (event) => |
| 2582 | initial.machineIds.includes(event.machineId!) && |
| 2583 | (event.path.endsWith("/cordon") || |
| 2584 | event.path.endsWith("/stop") || |
| 2585 | (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path))), |
| 2586 | ), |
| 2587 | ).toBe(false); |
| 2588 | expect( |
| 2589 | proxy.events.some( |
| 2590 | (event) => event.phase === "active" || event.phase === "retiring", |
| 2591 | ), |
| 2592 | ).toBe(false); |
| 2593 | expect( |
| 2594 | proxy.events.some( |
| 2595 | (event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path), |
| 2596 | ), |
| 2597 | ).toBe(false); |
| 2598 | const candidateIds = candidates.map((machine) => machine.id!).sort(); |
| 2599 | yield* Effect.sync(proxy.clear); |
| 2600 | const recovered = yield* deployWorker(stack, "two", props); |
| 2601 | expect([...recovered.machineIds].sort()).toEqual(candidateIds); |
| 2602 | yield* assertCommitted(initial.appName, recovered.machineIds); |
| 2603 | } finally { |
| 2604 | yield* Effect.sync(() => { |
| 2605 | endpoint = undefined; |
| 2606 | proxy.clear(); |
| 2607 | }); |
| 2608 | } |
| 2609 | yield* stack.destroy(); |
| 2610 | yield* assertAppGone(initial.appName); |
| 2611 | }).pipe( |
| 2612 | Effect.scoped, |
| 2613 | Effect.ensuring( |
| 2614 | Effect.sync(() => { |
| 2615 | endpoint = undefined; |
| 2616 | }), |
| 2617 | ), |
| 2618 | ), |
| 2619 | { timeout: 900_000 }, |
| 2620 | ); |
| 2621 | }, |
| 2622 | ); |
| 2623 | |
| 2624 | describe.sequential( |
| 2625 | "health transport faults", |
| 2626 | { |
| 2627 | tags: [ |
| 2628 | "provider:fly", |
| 2629 | "provider:fly:app", |
| 2630 | "provider:fly:machine", |
| 2631 | "provider:fly:service", |
| 2632 | "live", |
| 2633 | ], |
| 2634 | }, |
| 2635 | () => { |
| 2636 | let endpoint: string | undefined; |
| 2637 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 2638 | |
| 2639 | test.provider( |
| 2640 | "S07 health polling recovers a lost real GET and exhausts a persistent connection cut", |
| 2641 | (stack) => |
| 2642 | Effect.gen(function* () { |
| 2643 | yield* stack.destroy(); |
| 2644 | const initial = yield* deployWorker(stack, "one", { |
| 2645 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 2646 | }); |
| 2647 | const observed = yield* machines.getMachine({ |
| 2648 | app_name: initial.appName, |
| 2649 | machine_id: initial.machineId, |
| 2650 | }); |
| 2651 | const proxy = yield* transportProxy(); |
| 2652 | const match = (event: { method: string; path: string }) => |
| 2653 | event.method === "GET" && event.path.endsWith(`/machines/${initial.machineId}`); |
| 2654 | yield* Effect.sync(() => { |
| 2655 | endpoint = proxy.url; |
| 2656 | proxy.arm({ match, action: "drop-response", remaining: 1 }); |
| 2657 | }); |
| 2658 | try { |
| 2659 | const healthy = yield* waitHealthy( |
| 2660 | initial.appName, |
| 2661 | observed, |
| 2662 | 30_000, |
| 2663 | observed.config!, |
| 2664 | ); |
| 2665 | expect(healthy.id).toBe(initial.machineId); |
| 2666 | expect(healthy.instance_id).toBe(observed.instance_id); |
| 2667 | expect(healthy.checks?.every((check) => check.status === "passing")).toBe(true); |
| 2668 | expect( |
| 2669 | proxy.events.filter((event) => event.stage === "dropped" && event.status === 200), |
| 2670 | ).toHaveLength(1); |
| 2671 | yield* Effect.sync(() => |
| 2672 | proxy.arm({ match, action: "cut-request", remaining: Infinity }), |
| 2673 | ); |
| 2674 | let attempts = 0; |
| 2675 | const client = yield* HttpClient.HttpClient; |
| 2676 | const observedClient = HttpClient.tapRequest(client, (request) => |
| 2677 | Effect.sync(() => { |
| 2678 | if ( |
| 2679 | request.method === "GET" && |
| 2680 | request.url.endsWith(`/machines/${initial.machineId}`) |
| 2681 | ) { |
| 2682 | attempts++; |
| 2683 | } |
| 2684 | }), |
| 2685 | ); |
| 2686 | const started = yield* Clock.currentTimeMillis; |
| 2687 | const failed = yield* waitHealthy( |
| 2688 | initial.appName, |
| 2689 | healthy, |
| 2690 | 8_000, |
| 2691 | healthy.config!, |
| 2692 | ).pipe(Effect.provideService(HttpClient.HttpClient, observedClient), Effect.result); |
| 2693 | expect((yield* Clock.currentTimeMillis) - started).toBeLessThanOrEqual(12_000); |
| 2694 | expect(Result.isFailure(failed)).toBe(true); |
| 2695 | if (Result.isFailure(failed)) { |
| 2696 | expect(failed.failure).toMatchObject({ |
| 2697 | _tag: "Fly.ReplicaChecksNotPassing", |
| 2698 | appName: initial.appName, |
| 2699 | machineId: initial.machineId, |
| 2700 | }); |
| 2701 | } |
| 2702 | expect( |
| 2703 | proxy.events.filter((event) => event.stage === "cut").length, |
| 2704 | ).toBeGreaterThan(0); |
| 2705 | // Count logical client calls independently of Bun's physical GET retries. |
| 2706 | expect(attempts).toBeGreaterThan(0); |
| 2707 | expect(attempts).toBeLessThanOrEqual(11); |
| 2708 | const live = yield* census(initial.appName); |
| 2709 | expect(live.map((machine) => machine.id)).toEqual(initial.machineIds); |
| 2710 | expect(live[0]!.cordoned).toBe(false); |
| 2711 | } finally { |
| 2712 | yield* Effect.sync(() => { |
| 2713 | endpoint = undefined; |
| 2714 | proxy.clear(); |
| 2715 | }); |
| 2716 | } |
| 2717 | yield* stack.destroy(); |
| 2718 | yield* assertAppGone(initial.appName); |
| 2719 | }).pipe( |
| 2720 | Effect.scoped, |
| 2721 | Effect.ensuring( |
| 2722 | Effect.sync(() => { |
| 2723 | endpoint = undefined; |
| 2724 | }), |
| 2725 | ), |
| 2726 | ), |
| 2727 | { timeout: 180_000 }, |
| 2728 | ); |
| 2729 | }, |
| 2730 | ); |
| 2731 | |
| 2732 | describe.sequential( |
| 2733 | "failures", |
| 2734 | { |
| 2735 | tags: [ |
| 2736 | "provider:fly", |
| 2737 | "provider:fly:app", |
| 2738 | "provider:fly:machine", |
| 2739 | "provider:fly:service", |
| 2740 | "live", |
| 2741 | ], |
| 2742 | }, |
| 2743 | () => { |
| 2744 | const { test } = Test.make({ providers: Fly.providers() }); |
| 2745 | |
| 2746 | test.provider( |
| 2747 | "F01 a rejected candidate create preserves the active generation", |
| 2748 | (stack) => |
| 2749 | Effect.gen(function* () { |
| 2750 | yield* stack.destroy(); |
| 2751 | const deploy = (image: string) => |
| 2752 | stack.deploy( |
| 2753 | Effect.gen(function* () { |
| 2754 | const app = yield* Fly.App("Site"); |
| 2755 | return yield* Fly.Machine("Worker", { |
| 2756 | app, |
| 2757 | image, |
| 2758 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 2759 | shutdown: { signal: "SIGQUIT", timeout: "10 seconds" }, |
| 2760 | checks: { |
| 2761 | ready: { type: "http", port: 80, path: "/", interval: "2s", timeout: "1s" }, |
| 2762 | }, |
| 2763 | }); |
| 2764 | }), |
| 2765 | ); |
| 2766 | const initial = yield* deploy("nginx:alpine"); |
| 2767 | const failed = yield* deploy("nginx:alchemy-bluegreen-nonexistent-image").pipe( |
| 2768 | Effect.result, |
| 2769 | ); |
| 2770 | expect(Result.isFailure(failed)).toBe(true); |
| 2771 | if (Result.isFailure(failed)) |
| 2772 | expect(failed.failure).toMatchObject({ _tag: "BadRequest" }); |
| 2773 | const live = (yield* machines.listMachines({ app_name: initial.appName })).filter( |
| 2774 | (machine) => machine.state !== "destroyed", |
| 2775 | ); |
| 2776 | expect(live.map((machine) => machine.id)).toEqual(initial.machineIds); |
| 2777 | expect(live[0]?.state).toBe("started"); |
| 2778 | expect(live[0]?.cordoned).toBe(false); |
| 2779 | expect(live[0]?.config?.metadata?.["alchemy.phase"]).toBe("active"); |
| 2780 | yield* stack.destroy(); |
| 2781 | yield* assertAppGone(initial.appName); |
| 2782 | }), |
| 2783 | { timeout: 180_000 }, |
| 2784 | ); |
| 2785 | }, |
| 2786 | ); |
| 2787 | }); |
| 2788 | describe.sequential("shard 3", () => { |
| 2789 | describe.sequential( |
| 2790 | "managed HTTP services", |
| 2791 | { |
| 2792 | tags: [ |
| 2793 | "provider:fly", |
| 2794 | "provider:fly:app", |
| 2795 | "provider:fly:ipassignment", |
| 2796 | "provider:fly:machine", |
| 2797 | "provider:fly:redis", |
| 2798 | "provider:fly:secret", |
| 2799 | "provider:fly:service", |
| 2800 | "live", |
| 2801 | ], |
| 2802 | }, |
| 2803 | () => { |
| 2804 | const { test } = Test.make({ providers: Fly.providers() }); |
| 2805 | |
| 2806 | describe.sequential("Fly live HTTP and worker drain", () => { |
| 2807 | for (const policy of [ |
| 2808 | { |
| 2809 | name: "managed old60/new10 SIGTERM", |
| 2810 | raw: false, |
| 2811 | old: "60 seconds", |
| 2812 | next: "10 seconds", |
| 2813 | signal: "SIGTERM", |
| 2814 | nextSignal: "SIGINT", |
| 2815 | rawSignal: undefined, |
| 2816 | nextRawSignal: undefined, |
| 2817 | delay: 12_000, |
| 2818 | }, |
| 2819 | { |
| 2820 | name: "managed old10/new60 SIGINT", |
| 2821 | raw: false, |
| 2822 | old: "10 seconds", |
| 2823 | next: "60 seconds", |
| 2824 | signal: "SIGINT", |
| 2825 | nextSignal: "SIGTERM", |
| 2826 | rawSignal: undefined, |
| 2827 | nextRawSignal: undefined, |
| 2828 | delay: 3_000, |
| 2829 | }, |
| 2830 | { |
| 2831 | name: "raw old60/new10 SIGQUIT to SIGTERM", |
| 2832 | raw: true, |
| 2833 | old: "60 seconds", |
| 2834 | next: "10 seconds", |
| 2835 | signal: "SIGTERM", |
| 2836 | nextSignal: "SIGTERM", |
| 2837 | rawSignal: "SIGQUIT", |
| 2838 | nextRawSignal: "SIGTERM", |
| 2839 | delay: 12_000, |
| 2840 | }, |
| 2841 | { |
| 2842 | name: "raw old10/new60 SIGTERM to SIGQUIT", |
| 2843 | raw: true, |
| 2844 | old: "10 seconds", |
| 2845 | next: "60 seconds", |
| 2846 | signal: "SIGTERM", |
| 2847 | nextSignal: "SIGTERM", |
| 2848 | rawSignal: "SIGTERM", |
| 2849 | nextRawSignal: "SIGQUIT", |
| 2850 | delay: 3_000, |
| 2851 | }, |
| 2852 | ] as const) { |
| 2853 | test.provider( |
| 2854 | `R01/R02/R03/R04 ${policy.name} preserves traffic, jobs and streaming after the actual stop barrier`, |
| 2855 | (stack) => |
| 2856 | Effect.gen(function* () { |
| 2857 | yield* stack.destroy(); |
| 2858 | const scenario = yield* makeScenario(stack); |
| 2859 | yield* Effect.gen(function* () { |
| 2860 | const first = yield* scenario.deploy({ |
| 2861 | raw: policy.raw, |
| 2862 | version: "one", |
| 2863 | timeout: policy.old, |
| 2864 | signal: policy.signal, |
| 2865 | rawSignal: policy.rawSignal, |
| 2866 | afterSignalMs: policy.delay, |
| 2867 | }); |
| 2868 | const oldWorker = yield* requireValue(first.worker, "initial worker output"); |
| 2869 | const oldId = oldWorker.machineId; |
| 2870 | const url = `https://${first.workerApp.appName}.fly.dev`; |
| 2871 | const get = (route: string) => |
| 2872 | HttpClient.get(`${url}${route}`, { headers: { connection: "close" } }).pipe( |
| 2873 | Effect.flatMap((response) => |
| 2874 | response.status === 200 |
| 2875 | ? response.text |
| 2876 | : Effect.fail(new Error(`HTTP ${response.status}`)), |
| 2877 | ), |
| 2878 | Effect.timeout(route === "/" ? "5 seconds" : "240 seconds"), |
| 2879 | ); |
| 2880 | expect( |
| 2881 | JSON.parse( |
| 2882 | yield* get("/").pipe( |
| 2883 | Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") }), |
| 2884 | ), |
| 2885 | ), |
| 2886 | ).toEqual({ machine: oldId, version: "one" }); |
| 2887 | const observed = yield* machines.getMachine({ |
| 2888 | app_name: first.workerApp.appName, |
| 2889 | machine_id: oldId, |
| 2890 | }); |
| 2891 | expect(observed.config?.stop_config?.signal).toBe( |
| 2892 | policy.rawSignal ?? policy.signal, |
| 2893 | ); |
| 2894 | expect( |
| 2895 | policy.old === "60 seconds" |
| 2896 | ? ["60000ms", "60s", "1m", "1m0s"] |
| 2897 | : ["10000ms", "10s"], |
| 2898 | ).toContain(observed.config?.stop_config?.timeout); |
| 2899 | yield* scenario.call(first.ledgerUrl, "enqueue", ["in-flight-job", "hold"]); |
| 2900 | yield* scenario.wait(first.ledgerUrl, (ledger) => |
| 2901 | ledger.events.some( |
| 2902 | (event) => |
| 2903 | event.machine === oldId && |
| 2904 | event.event === "claimed" && |
| 2905 | event.job === "in-flight-job", |
| 2906 | ), |
| 2907 | ); |
| 2908 | const slow = yield* get("/hold").pipe(Effect.forkScoped); |
| 2909 | const streamed = yield* get("/stream").pipe(Effect.forkScoped); |
| 2910 | yield* scenario.wait( |
| 2911 | first.ledgerUrl, |
| 2912 | (ledger) => |
| 2913 | ledger.events.filter( |
| 2914 | (event) => event.machine === oldId && event.event === "request-started", |
| 2915 | ).length === 2, |
| 2916 | ); |
| 2917 | const responses = yield* Ref.make< |
| 2918 | Array<{ machine: string; version: string } | { failure: string }> |
| 2919 | >([]); |
| 2920 | const finished = yield* Ref.make(false); |
| 2921 | yield* Effect.addFinalizer(() => |
| 2922 | Ref.get(responses).pipe( |
| 2923 | Effect.flatMap((samples) => |
| 2924 | Effect.logInfo("Unretried public traffic samples", samples), |
| 2925 | ), |
| 2926 | ), |
| 2927 | ); |
| 2928 | const traffic = yield* Stream.range(0, 599).pipe( |
| 2929 | Stream.mapEffect(() => |
| 2930 | get("/").pipe( |
| 2931 | Effect.result, |
| 2932 | Effect.flatMap((result) => |
| 2933 | Ref.update(responses, (values) => [ |
| 2934 | ...values, |
| 2935 | Result.isSuccess(result) |
| 2936 | ? (JSON.parse(result.success) as { |
| 2937 | machine: string; |
| 2938 | version: string; |
| 2939 | }) |
| 2940 | : { failure: String(result.failure) }, |
| 2941 | ]), |
| 2942 | ), |
| 2943 | Effect.andThen(Effect.sleep("500 millis")), |
| 2944 | Effect.andThen(Ref.get(finished)), |
| 2945 | ), |
| 2946 | ), |
| 2947 | Stream.takeUntil((done) => done), |
| 2948 | Stream.runDrain, |
| 2949 | Effect.forkScoped, |
| 2950 | ); |
| 2951 | const second = yield* scenario.deploy({ |
| 2952 | raw: policy.raw, |
| 2953 | version: "two", |
| 2954 | timeout: policy.next, |
| 2955 | signal: policy.nextSignal, |
| 2956 | rawSignal: policy.nextRawSignal, |
| 2957 | afterSignalMs: 1000, |
| 2958 | }); |
| 2959 | const newWorker = yield* requireValue( |
| 2960 | second.worker, |
| 2961 | "replacement worker output", |
| 2962 | ); |
| 2963 | expect(JSON.parse(yield* Fiber.join(slow))).toEqual({ |
| 2964 | machine: oldId, |
| 2965 | version: "one", |
| 2966 | }); |
| 2967 | // The fixture pads quiet gaps with blank heartbeat lines so Fly's proxy |
| 2968 | // never sees a 60s-idle connection; the payload itself must be intact. |
| 2969 | const body = (yield* Fiber.join(streamed)).replace(/^\n+/gm, ""); |
| 2970 | const expected = "first\n".repeat(32768) + "last\n".repeat(32768); |
| 2971 | const hashes = yield* Effect.sync(() => |
| 2972 | [body, expected].map((value) => |
| 2973 | createHash("sha256").update(value).digest("hex"), |
| 2974 | ), |
| 2975 | ); |
| 2976 | expect(body.length).toBe(360448); |
| 2977 | expect(hashes[0]).toBe(hashes[1]); |
| 2978 | expect(JSON.parse(yield* get("/"))).toEqual({ |
| 2979 | machine: newWorker.machineId, |
| 2980 | version: "two", |
| 2981 | }); |
| 2982 | yield* Effect.sleep("600 millis"); |
| 2983 | yield* Ref.set(finished, true); |
| 2984 | yield* Fiber.join(traffic); |
| 2985 | const samples = yield* Ref.get(responses); |
| 2986 | expect(samples.filter((sample) => "failure" in sample)).toEqual([]); |
| 2987 | expect( |
| 2988 | samples.some( |
| 2989 | (sample) => |
| 2990 | "version" in sample && |
| 2991 | sample.version === "one" && |
| 2992 | sample.machine === oldId, |
| 2993 | ), |
| 2994 | ).toBe(true); |
| 2995 | expect( |
| 2996 | samples.some( |
| 2997 | (sample) => |
| 2998 | "version" in sample && |
| 2999 | sample.version === "two" && |
| 3000 | sample.machine === newWorker.machineId, |
| 3001 | ), |
| 3002 | ).toBe(true); |
| 3003 | yield* assertReplacement(first.workerApp.appName, oldId, newWorker.machineId); |
| 3004 | const ledger = yield* scenario.settle(first.ledgerUrl); |
| 3005 | assertStopped(ledger.events, oldId); |
| 3006 | assertOrder(ledger.events, oldId, "request-finalized", "shared-closed"); |
| 3007 | assertOrder(ledger.events, oldId, "client-released", "shared-closed"); |
| 3008 | const stop = yield* requireValue( |
| 3009 | ledger.events.find( |
| 3010 | (event) => event.machine === oldId && event.event === "stop-started", |
| 3011 | ), |
| 3012 | "old worker stop event", |
| 3013 | ); |
| 3014 | if (policy.raw) expect(stop.signal).toBe(policy.rawSignal); |
| 3015 | const completed = ledger.events.filter( |
| 3016 | (event) => event.machine === oldId && event.event === "response-finished", |
| 3017 | ); |
| 3018 | expect(completed).toHaveLength(2); |
| 3019 | for (const event of completed) { |
| 3020 | expect(event.at - stop.at).toBeGreaterThan(policy.delay - 500); |
| 3021 | } |
| 3022 | const ack = yield* requireValue( |
| 3023 | ledger.events.find( |
| 3024 | (event) => |
| 3025 | event.machine === oldId && |
| 3026 | event.event === "acked" && |
| 3027 | event.job === "in-flight-job", |
| 3028 | ), |
| 3029 | "in-flight job ACK event", |
| 3030 | ); |
| 3031 | expect(ack.ack).toBe(1); |
| 3032 | expect(ack.at - stop.at).toBeGreaterThan(policy.delay - 500); |
| 3033 | if (policy.old === "10 seconds") expect(ack.at - stop.at).toBeLessThan(10_000); |
| 3034 | }).pipe(Effect.scoped, Effect.ensuring(scenario.cleanup)); |
| 3035 | }), |
| 3036 | { |
| 3037 | // Includes cold publication, both readiness phases, proxy overlap, post-signal work and census. |
| 3038 | timeout: 480_000, |
| 3039 | }, |
| 3040 | ); |
| 3041 | } |
| 3042 | }); |
| 3043 | }, |
| 3044 | ); |
| 3045 | |
| 3046 | describe.sequential( |
| 3047 | "legacy compatibility", |
| 3048 | { |
| 3049 | tags: [ |
| 3050 | "provider:fly", |
| 3051 | "provider:fly:app", |
| 3052 | "provider:fly:machine", |
| 3053 | "provider:fly:service", |
| 3054 | "live", |
| 3055 | ], |
| 3056 | }, |
| 3057 | () => { |
| 3058 | const file = "test/Fly/BlueGreen.test.ts"; |
| 3059 | let endpoint: string | undefined; |
| 3060 | let stops: StopRequest[] = []; |
| 3061 | const { test } = Test.make({ |
| 3062 | providers: observeStops( |
| 3063 | () => endpoint, |
| 3064 | (request) => stops.push(request), |
| 3065 | ), |
| 3066 | }); |
| 3067 | |
| 3068 | const runningLegacy = (appName: string, id: string) => |
| 3069 | machines.getMachine({ app_name: appName, machine_id: id }).pipe( |
| 3070 | Effect.repeat({ |
| 3071 | schedule: Schedule.spaced("2 seconds"), |
| 3072 | times: 45, |
| 3073 | until: (machine) => |
| 3074 | machine.state === "started" && |
| 3075 | machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined && |
| 3076 | machine.config?.metadata?.["alchemy.generation"] === undefined && |
| 3077 | machine.config?.stop_config === undefined, |
| 3078 | }), |
| 3079 | Effect.timeout("120 seconds"), |
| 3080 | Effect.tap((machine) => |
| 3081 | Effect.sync(() => { |
| 3082 | expect(machine.state).toBe("started"); |
| 3083 | expect(machine.config?.stop_config).toBeUndefined(); |
| 3084 | expect(machine.config?.metadata?.["alchemy.deployment-protocol"]).toBeUndefined(); |
| 3085 | expect(machine.config?.metadata?.["alchemy.generation"]).toBeUndefined(); |
| 3086 | }), |
| 3087 | ), |
| 3088 | ); |
| 3089 | |
| 3090 | describe.sequential("live legacy protocol compatibility models", () => { |
| 3091 | for (const runtimeTimeoutMs of [undefined, 10_000, 60_000]) { |
| 3092 | test.provider( |
| 3093 | `F14 live legacy missing stop_config ${runtimeTimeoutMs === undefined ? "preserves raw-image defaults" : `uses ${runtimeTimeoutMs}ms runtime-env fallback`}`, |
| 3094 | (stack) => |
| 3095 | Effect.gen(function* () { |
| 3096 | yield* stack.destroy(); |
| 3097 | const site = yield* stack.deploy(Fly.App("Site")); |
| 3098 | try { |
| 3099 | const initial = yield* deployWorker(stack, "one", { |
| 3100 | deploy: { strategy: "rolling" }, |
| 3101 | shutdown: undefined, |
| 3102 | }); |
| 3103 | yield* writeLegacyProtocol(site.appName, initial.machineId, runtimeTimeoutMs); |
| 3104 | const legacy = yield* runningLegacy(site.appName, initial.machineId); |
| 3105 | expect(legacy.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS).toBe( |
| 3106 | runtimeTimeoutMs?.toString(), |
| 3107 | ); |
| 3108 | const policy = yield* predecessorShutdown(legacy); |
| 3109 | if (runtimeTimeoutMs === undefined) { |
| 3110 | expect(policy.signal).toBeUndefined(); |
| 3111 | expect(policy.timeout).toBeUndefined(); |
| 3112 | } else { |
| 3113 | expect(policy.signal).toBe("SIGTERM"); |
| 3114 | expect(policy.timeout).toBe(`${runtimeTimeoutMs}ms`); |
| 3115 | expect(policy.timeoutMs).toBe(runtimeTimeoutMs); |
| 3116 | } |
| 3117 | const proxy = yield* transportProxy(); |
| 3118 | yield* Effect.sync(() => { |
| 3119 | endpoint = proxy.url; |
| 3120 | stops = []; |
| 3121 | }); |
| 3122 | const upgraded = yield* deployWorker(stack, "two", { |
| 3123 | deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" }, |
| 3124 | }); |
| 3125 | expect(upgraded.machineId).not.toBe(initial.machineId); |
| 3126 | yield* assertCommitted(site.appName, upgraded.machineIds); |
| 3127 | const stop = stops.filter((request) => request.machineId === initial.machineId); |
| 3128 | expect(stop).toHaveLength(1); |
| 3129 | expect(stop[0]!.signal).toBe( |
| 3130 | runtimeTimeoutMs === undefined ? undefined : "SIGTERM", |
| 3131 | ); |
| 3132 | expect(stop[0]!.timeout).toBe( |
| 3133 | runtimeTimeoutMs === undefined ? undefined : `${runtimeTimeoutMs}ms`, |
| 3134 | ); |
| 3135 | expect( |
| 3136 | proxy.events.some( |
| 3137 | (event) => |
| 3138 | event.stage === "completed" && |
| 3139 | event.machineId === initial.machineId && |
| 3140 | event.path.endsWith("/stop") && |
| 3141 | event.status! >= 200 && |
| 3142 | event.status! < 300, |
| 3143 | ), |
| 3144 | ).toBe(true); |
| 3145 | expect( |
| 3146 | proxy.events.some( |
| 3147 | (event) => |
| 3148 | event.stage === "completed" && |
| 3149 | event.machineId === initial.machineId && |
| 3150 | event.method === "DELETE" && |
| 3151 | /\/machines\/[^/]+$/.test(event.path) && |
| 3152 | event.status! >= 200 && |
| 3153 | event.status! < 300, |
| 3154 | ), |
| 3155 | ).toBe(true); |
| 3156 | const same = yield* deployWorker(stack, "two", { |
| 3157 | deploy: { strategy: "bluegreen", healthTimeout: "65 seconds" }, |
| 3158 | }); |
| 3159 | expect(same.machineIds).toEqual(upgraded.machineIds); |
| 3160 | } finally { |
| 3161 | yield* Effect.sync(() => { |
| 3162 | endpoint = undefined; |
| 3163 | stops = []; |
| 3164 | }); |
| 3165 | yield* stack.destroy(); |
| 3166 | yield* assertAppGone(site.appName); |
| 3167 | } |
| 3168 | }).pipe(Effect.scoped), |
| 3169 | { timeout: 900_000 }, |
| 3170 | ); |
| 3171 | } |
| 3172 | |
| 3173 | for (const affected of [1, 2]) { |
| 3174 | test.provider( |
| 3175 | `F14 unknown metadata protocol on ${affected} of 2 real Machines safely refuses without mutation`, |
| 3176 | (stack) => |
| 3177 | Effect.gen(function* () { |
| 3178 | yield* stack.destroy(); |
| 3179 | const site = yield* stack.deploy(Fly.App("Site")); |
| 3180 | try { |
| 3181 | const initial = yield* deployWorker(stack, "one", { count: 2 }); |
| 3182 | for (const id of initial.machineIds.slice(0, affected)) { |
| 3183 | const target = { app_name: site.appName, machine_id: id }; |
| 3184 | const current = yield* machines.getMachine(target); |
| 3185 | yield* machines.patchMachineMetadata({ |
| 3186 | ...target, |
| 3187 | metadata: { |
| 3188 | ...current.config!.metadata, |
| 3189 | "alchemy.deployment-protocol": "future-unknown", |
| 3190 | }, |
| 3191 | }); |
| 3192 | } |
| 3193 | const before = yield* census(site.appName); |
| 3194 | expect( |
| 3195 | before.filter( |
| 3196 | (machine) => |
| 3197 | machine.config?.metadata?.["alchemy.deployment-protocol"] === |
| 3198 | "future-unknown", |
| 3199 | ), |
| 3200 | ).toHaveLength(affected); |
| 3201 | const metadata = before[0]!.config!.metadata!; |
| 3202 | const read = yield* observeReplicaSet({ |
| 3203 | appName: site.appName, |
| 3204 | id: "Worker", |
| 3205 | type: "Fly.Machine", |
| 3206 | fqn: metadata["alchemy.fqn"]!, |
| 3207 | resourceInstanceId: metadata["alchemy.instance"]!, |
| 3208 | machineIds: initial.machineIds, |
| 3209 | }); |
| 3210 | expect(read?.rolloutPending).toBe(true); |
| 3211 | expect(read?.machineIds).toEqual([]); |
| 3212 | const proxy = yield* transportProxy(); |
| 3213 | yield* Effect.sync(() => { |
| 3214 | endpoint = proxy.url; |
| 3215 | }); |
| 3216 | const result = yield* deployWorker(stack, "two", { count: 2 }).pipe( |
| 3217 | Effect.timeout("180 seconds"), |
| 3218 | Effect.result, |
| 3219 | ); |
| 3220 | expect(Result.isFailure(result)).toBe(true); |
| 3221 | if (Result.isFailure(result)) { |
| 3222 | expect(result.failure).toMatchObject({ |
| 3223 | _tag: "Fly.DeploymentRecoveryAmbiguous", |
| 3224 | }); |
| 3225 | } |
| 3226 | expect( |
| 3227 | proxy.events.some( |
| 3228 | (event) => |
| 3229 | event.stage === "request" && |
| 3230 | event.method !== "GET" && |
| 3231 | !event.path.endsWith("/lease"), |
| 3232 | ), |
| 3233 | ).toBe(false); |
| 3234 | const after = yield* census(site.appName); |
| 3235 | expect(after.map((machine) => machine.id).sort()).toEqual( |
| 3236 | initial.machineIds.slice().sort(), |
| 3237 | ); |
| 3238 | for (const machine of after) { |
| 3239 | const previous = before.find((item) => item.id === machine.id)!; |
| 3240 | expect(machine.instance_id).toBe(previous.instance_id); |
| 3241 | expect(machine.config).toEqual(previous.config); |
| 3242 | expect(machine.cordoned).toBe(false); |
| 3243 | expect(machine.state).toBe("started"); |
| 3244 | } |
| 3245 | } finally { |
| 3246 | yield* Effect.sync(() => { |
| 3247 | endpoint = undefined; |
| 3248 | }); |
| 3249 | yield* stack.destroy(); |
| 3250 | yield* assertAppGone(site.appName); |
| 3251 | } |
| 3252 | }).pipe(Effect.scoped), |
| 3253 | { timeout: 600_000 }, |
| 3254 | ); |
| 3255 | } |
| 3256 | |
| 3257 | test.provider( |
| 3258 | "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence", |
| 3259 | (stack) => |
| 3260 | Effect.gen(function* () { |
| 3261 | yield* stack.destroy(); |
| 3262 | const site = yield* stack.deploy(Fly.App("Site")); |
| 3263 | try { |
| 3264 | const first = yield* deployWorker(stack, "one", { count: 2 }); |
| 3265 | yield* writeLegacyProtocol(site.appName, first.machineIds[0]!, 60_000); |
| 3266 | yield* runningLegacy(site.appName, first.machineIds[0]!); |
| 3267 | const mixed = yield* census(site.appName); |
| 3268 | expect( |
| 3269 | mixed.filter( |
| 3270 | (machine) => |
| 3271 | machine.config?.metadata?.["alchemy.deployment-protocol"] === "1", |
| 3272 | ), |
| 3273 | ).toHaveLength(1); |
| 3274 | expect( |
| 3275 | mixed.filter( |
| 3276 | (machine) => |
| 3277 | machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined, |
| 3278 | ), |
| 3279 | ).toHaveLength(1); |
| 3280 | const proxy = yield* transportProxy(); |
| 3281 | try { |
| 3282 | yield* Effect.sync(() => |
| 3283 | proxy.arm({ |
| 3284 | match: (event) => |
| 3285 | event.method === "POST" && event.path.endsWith("/machines"), |
| 3286 | action: "hold-response", |
| 3287 | remaining: 1, |
| 3288 | }), |
| 3289 | ); |
| 3290 | yield* Effect.gen(function* () { |
| 3291 | const actor = yield* engineActor( |
| 3292 | stack, |
| 3293 | "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence", |
| 3294 | file, |
| 3295 | proxy.url, |
| 3296 | ); |
| 3297 | const upgrade = yield* deployWorker(actor, "two", { count: 2 }).pipe( |
| 3298 | Effect.scoped, |
| 3299 | Effect.forkScoped, |
| 3300 | ); |
| 3301 | yield* proxy.wait( |
| 3302 | (event) => |
| 3303 | event.stage === "held" && event.status! >= 200 && event.status! < 300, |
| 3304 | ); |
| 3305 | const competing = yield* writeLegacyProtocol( |
| 3306 | site.appName, |
| 3307 | first.machineIds[1]!, |
| 3308 | ).pipe(Effect.result); |
| 3309 | expect(Result.isFailure(competing)).toBe(true); |
| 3310 | if (Result.isFailure(competing)) |
| 3311 | expect(competing.failure._tag).toBe("Conflict"); |
| 3312 | expect( |
| 3313 | (yield* census(site.appName)).filter((machine) => |
| 3314 | first.machineIds.includes(machine.id!), |
| 3315 | ), |
| 3316 | ).toHaveLength(2); |
| 3317 | yield* Effect.sync(() => { |
| 3318 | proxy.clear(); |
| 3319 | proxy.release(); |
| 3320 | }); |
| 3321 | const upgraded = yield* Fiber.join(upgrade).pipe( |
| 3322 | Effect.timeout("300 seconds"), |
| 3323 | ); |
| 3324 | yield* assertCommitted(site.appName, upgraded.machineIds); |
| 3325 | // A lease-aware legacy writer can still change a successor after lease release. |
| 3326 | yield* writeLegacyProtocol(site.appName, upgraded.machineIds[0]!, 60_000); |
| 3327 | yield* runningLegacy(site.appName, upgraded.machineIds[0]!); |
| 3328 | const partial = yield* census(site.appName); |
| 3329 | expect(partial.map((machine) => machine.id).sort()).toEqual( |
| 3330 | upgraded.machineIds.slice().sort(), |
| 3331 | ); |
| 3332 | expect( |
| 3333 | partial.filter( |
| 3334 | (machine) => |
| 3335 | machine.config?.metadata?.["alchemy.deployment-protocol"] === undefined, |
| 3336 | ), |
| 3337 | ).toHaveLength(1); |
| 3338 | const resumed = yield* engineActor( |
| 3339 | stack, |
| 3340 | "F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence", |
| 3341 | file, |
| 3342 | ); |
| 3343 | const recovered = yield* deployWorker(resumed, "three", { count: 2 }).pipe( |
| 3344 | Effect.scoped, |
| 3345 | ); |
| 3346 | expect( |
| 3347 | recovered.machineIds.every((id) => !upgraded.machineIds.includes(id)), |
| 3348 | ).toBe(true); |
| 3349 | yield* assertCommitted(site.appName, recovered.machineIds); |
| 3350 | yield* Effect.logInfo("Partial-upgrade model boundary", { |
| 3351 | model: |
| 3352 | "native lease-aware legacy protocol writer, not a literal old binary", |
| 3353 | globalFence: false, |
| 3354 | recoveredCount: recovered.machineIds.length, |
| 3355 | }); |
| 3356 | }).pipe( |
| 3357 | Effect.ensuring( |
| 3358 | Effect.sync(() => { |
| 3359 | proxy.clear(); |
| 3360 | proxy.release(); |
| 3361 | }), |
| 3362 | ), |
| 3363 | Effect.scoped, |
| 3364 | ); |
| 3365 | } finally { |
| 3366 | yield* Effect.sync(() => { |
| 3367 | proxy.clear(); |
| 3368 | proxy.release(); |
| 3369 | }); |
| 3370 | } |
| 3371 | } finally { |
| 3372 | yield* stack.destroy(); |
| 3373 | yield* assertAppGone(site.appName); |
| 3374 | } |
| 3375 | }).pipe(Effect.scoped), |
| 3376 | { timeout: 900_000 }, |
| 3377 | ); |
| 3378 | }); |
| 3379 | }, |
| 3380 | ); |
| 3381 | |
| 3382 | describe.sequential( |
| 3383 | "health deadlines", |
| 3384 | { |
| 3385 | tags: [ |
| 3386 | "provider:fly", |
| 3387 | "provider:fly:app", |
| 3388 | "provider:fly:machine", |
| 3389 | "provider:fly:service", |
| 3390 | "live", |
| 3391 | ], |
| 3392 | }, |
| 3393 | () => { |
| 3394 | const { test } = Test.make({ providers: Fly.providers() }); |
| 3395 | |
| 3396 | for (const budget of [8_000, 90_000]) { |
| 3397 | test.provider( |
| 3398 | `S06 real delayed readiness beyond the default minute with ${budget}ms budget`, |
| 3399 | (stack) => |
| 3400 | Effect.gen(function* () { |
| 3401 | yield* stack.destroy(); |
| 3402 | const initial = yield* deployWorker(stack, "one"); |
| 3403 | const result = yield* deployWorker(stack, "two", { |
| 3404 | init: { exec: ["/bin/sh", "-c", "sleep 65; exec nginx -g 'daemon off;'"] }, |
| 3405 | checks: { ready: { ...checks.ready, gracePeriod: "65s" } }, |
| 3406 | deploy: { strategy: "bluegreen", healthTimeout: budget }, |
| 3407 | }).pipe(Effect.result); |
| 3408 | if (budget === 8_000) { |
| 3409 | expect(Result.isFailure(result)).toBe(true); |
| 3410 | if (Result.isFailure(result)) |
| 3411 | expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" }); |
| 3412 | const live = yield* census(initial.appName); |
| 3413 | expect(live.map((machine) => machine.id)).toEqual(initial.machineIds); |
| 3414 | expect(live[0]!.cordoned).toBe(false); |
| 3415 | } else { |
| 3416 | expect(Result.isSuccess(result)).toBe(true); |
| 3417 | if (Result.isSuccess(result)) { |
| 3418 | expect(result.success.machineId).not.toBe(initial.machineId); |
| 3419 | yield* assertCommitted(initial.appName, result.success.machineIds); |
| 3420 | } |
| 3421 | } |
| 3422 | yield* stack.destroy(); |
| 3423 | yield* assertAppGone(initial.appName); |
| 3424 | }), |
| 3425 | { timeout: 600_000 }, |
| 3426 | ); |
| 3427 | } |
| 3428 | }, |
| 3429 | ); |
| 3430 | |
| 3431 | describe.sequential( |
| 3432 | "validation", |
| 3433 | { |
| 3434 | tags: [ |
| 3435 | "provider:fly", |
| 3436 | "provider:fly:app", |
| 3437 | "provider:fly:machine", |
| 3438 | "provider:fly:service", |
| 3439 | "live", |
| 3440 | ], |
| 3441 | }, |
| 3442 | () => { |
| 3443 | let endpoint: string | undefined; |
| 3444 | const { test } = Test.make({ providers: throughProxy(() => endpoint) }); |
| 3445 | const invalid: Array<[string, Partial<Extract<MachineProps, { image: string }>>]> = [ |
| 3446 | ["direct volume", { mounts: [{ path: "/data", sizeGb: 1 }] }], |
| 3447 | ["skipLaunch", { skipLaunch: true }], |
| 3448 | ["autoDestroy", { autoDestroy: true }], |
| 3449 | ["restart=no", { restart: { policy: "no" } }], |
| 3450 | ["missing checks", { checks: {} }], |
| 3451 | ["unchecked public service", { services: [{ internalPort: 80, ports: [{ port: 80 }] }] }], |
| 3452 | ]; |
| 3453 | |
| 3454 | describe.sequential("pre-mutation validation", () => { |
| 3455 | for (const [label, props] of invalid) { |
| 3456 | test.provider( |
| 3457 | `${label === "direct volume" ? "S10" : "S11"} ${label} rejects before any Machine mutation`, |
| 3458 | (stack) => |
| 3459 | Effect.gen(function* () { |
| 3460 | yield* stack.destroy(); |
| 3461 | const app = yield* stack.deploy(Fly.App("Site")); |
| 3462 | const proxy = yield* transportProxy(); |
| 3463 | yield* Effect.sync(() => { |
| 3464 | endpoint = proxy.url; |
| 3465 | }); |
| 3466 | const failed = yield* stack |
| 3467 | .deploy( |
| 3468 | Effect.gen(function* () { |
| 3469 | const app = yield* Fly.App("Site"); |
| 3470 | return yield* Fly.Machine("Worker", { |
| 3471 | app, |
| 3472 | image: "nginx:alpine", |
| 3473 | checks, |
| 3474 | deploy: { strategy: "bluegreen" }, |
| 3475 | ...props, |
| 3476 | }); |
| 3477 | }), |
| 3478 | ) |
| 3479 | .pipe(Effect.result); |
| 3480 | expect(Result.isFailure(failed)).toBe(true); |
| 3481 | if (Result.isFailure(failed)) |
| 3482 | expect(failed.failure).toMatchObject({ _tag: "Fly.InvalidDeployment" }); |
| 3483 | expect( |
| 3484 | proxy.events.some( |
| 3485 | (event) => |
| 3486 | event.method !== "GET" && /\/(machines|volumes)(\/|$)/.test(event.path), |
| 3487 | ), |
| 3488 | ).toBe(false); |
| 3489 | expect(yield* census(app.appName)).toHaveLength(0); |
| 3490 | expect(yield* machines.listVolumes({ app_name: app.appName })).toHaveLength(0); |
| 3491 | yield* Effect.sync(() => { |
| 3492 | endpoint = undefined; |
| 3493 | }); |
| 3494 | yield* stack.destroy(); |
| 3495 | yield* assertAppGone(app.appName); |
| 3496 | }).pipe( |
| 3497 | Effect.scoped, |
| 3498 | Effect.ensuring( |
| 3499 | Effect.sync(() => { |
| 3500 | endpoint = undefined; |
| 3501 | }), |
| 3502 | ), |
| 3503 | ), |
| 3504 | { timeout: 180_000 }, |
| 3505 | ); |
| 3506 | } |
| 3507 | |
| 3508 | test.provider( |
| 3509 | "S10 resolved MountVolume binding rejects before Machine or Volume mutation", |
| 3510 | (stack) => |
| 3511 | Effect.gen(function* () { |
| 3512 | yield* stack.destroy(); |
| 3513 | const app = yield* stack.deploy(Fly.App("Site")); |
| 3514 | const proxy = yield* transportProxy(); |
| 3515 | yield* Effect.sync(() => { |
| 3516 | endpoint = proxy.url; |
| 3517 | }); |
| 3518 | const failed = yield* stack.deploy(MountedBlueGreen).pipe(Effect.result); |
| 3519 | expect(Result.isFailure(failed)).toBe(true); |
| 3520 | if (Result.isFailure(failed)) |
| 3521 | expect(failed.failure).toMatchObject({ _tag: "Fly.InvalidDeployment" }); |
| 3522 | expect( |
| 3523 | proxy.events.some( |
| 3524 | (event) => |
| 3525 | event.method !== "GET" && /\/(machines|volumes)(\/|$)/.test(event.path), |
| 3526 | ), |
| 3527 | ).toBe(false); |
| 3528 | expect(yield* census(app.appName)).toHaveLength(0); |
| 3529 | expect(yield* machines.listVolumes({ app_name: app.appName })).toHaveLength(0); |
| 3530 | yield* Effect.sync(() => { |
| 3531 | endpoint = undefined; |
| 3532 | }); |
| 3533 | yield* stack.destroy(); |
| 3534 | yield* assertAppGone(app.appName); |
| 3535 | }).pipe( |
| 3536 | Effect.scoped, |
| 3537 | Effect.ensuring( |
| 3538 | Effect.sync(() => { |
| 3539 | endpoint = undefined; |
| 3540 | }), |
| 3541 | ), |
| 3542 | ), |
| 3543 | { tags: ["provider:fly:volume"], timeout: 300_000 }, |
| 3544 | ); |
| 3545 | }); |
| 3546 | }, |
| 3547 | ); |
| 3548 | |
| 3549 | describe.sequential( |
| 3550 | "runtime secrets", |
| 3551 | { |
| 3552 | tags: [ |
| 3553 | "provider:fly", |
| 3554 | "provider:fly:app", |
| 3555 | "provider:fly:ipassignment", |
| 3556 | "provider:fly:machine", |
| 3557 | "provider:fly:secret", |
| 3558 | "provider:fly:service", |
| 3559 | "live", |
| 3560 | ], |
| 3561 | }, |
| 3562 | () => { |
| 3563 | const { test } = Test.make({ providers: Fly.providers() }); |
| 3564 | |
| 3565 | class WriterProbeFailed extends Data.TaggedError("WriterProbeFailed")<{ |
| 3566 | stage: "transport" | "status" | "decode"; |
| 3567 | status?: number; |
| 3568 | }> {} |
| 3569 | |
| 3570 | const Receipt = Schema.Struct({ |
| 3571 | version: Schema.Number, |
| 3572 | machineId: Schema.String, |
| 3573 | marker: Schema.Literals(["ready", "three"]), |
| 3574 | }); |
| 3575 | |
| 3576 | const probeWriter = ( |
| 3577 | appName: string, |
| 3578 | token: Redacted.Redacted<string>, |
| 3579 | method: "GET" | "POST", |
| 3580 | ) => |
| 3581 | Effect.gen(function* () { |
| 3582 | const client = yield* HttpClient.HttpClient; |
| 3583 | const url = `https://${appName}.fly.dev/writer`; |
| 3584 | const request = ( |
| 3585 | method === "POST" ? HttpClientRequest.post(url) : HttpClientRequest.get(url) |
| 3586 | ).pipe(HttpClientRequest.bearerToken(token)); |
| 3587 | const response = yield* client |
| 3588 | .execute(request) |
| 3589 | .pipe(Effect.mapError(() => new WriterProbeFailed({ stage: "transport" }))); |
| 3590 | if (response.status !== 200) { |
| 3591 | return yield* new WriterProbeFailed({ stage: "status", status: response.status }); |
| 3592 | } |
| 3593 | return yield* response.json.pipe( |
| 3594 | Effect.flatMap(Schema.decodeUnknownEffect(Receipt, { onExcessProperty: "error" })), |
| 3595 | Effect.mapError(() => new WriterProbeFailed({ stage: "decode" })), |
| 3596 | ); |
| 3597 | }).pipe( |
| 3598 | Effect.timeout("30 seconds"), |
| 3599 | Effect.catchTag("TimeoutError", () => |
| 3600 | Effect.fail(new WriterProbeFailed({ stage: "transport" })), |
| 3601 | ), |
| 3602 | ); |
| 3603 | |
| 3604 | const marker = (appName: string, machineId: string) => |
| 3605 | machines |
| 3606 | .execMachine({ |
| 3607 | app_name: appName, |
| 3608 | machine_id: machineId, |
| 3609 | command: ["cat", "/usr/share/nginx/html/marker"], |
| 3610 | timeout: 5, |
| 3611 | }) |
| 3612 | .pipe( |
| 3613 | Effect.map((response) => { |
| 3614 | const value = response.stdout?.trim(); |
| 3615 | return { |
| 3616 | code: response.exit_code, |
| 3617 | marker: value === "two" || value === "three" ? value : "missing", |
| 3618 | }; |
| 3619 | }), |
| 3620 | ); |
| 3621 | |
| 3622 | test.provider( |
| 3623 | "F12 deployed sibling runtime writer advances the shared vault during a held candidate create", |
| 3624 | (stack) => |
| 3625 | Effect.gen(function* () { |
| 3626 | yield* stack.destroy(); |
| 3627 | const proxy = yield* transportProxy(); |
| 3628 | let appName: string | undefined; |
| 3629 | yield* Effect.addFinalizer(() => |
| 3630 | stack.destroy().pipe( |
| 3631 | Effect.andThen(() => |
| 3632 | appName === undefined ? Effect.void : assertAppGone(appName), |
| 3633 | ), |
| 3634 | Effect.orDie, |
| 3635 | ), |
| 3636 | ); |
| 3637 | const actor = yield* engineActor( |
| 3638 | stack, |
| 3639 | "F12 deployed sibling runtime writer advances the shared vault during a held candidate create", |
| 3640 | "test/Fly/BlueGreen.test.ts", |
| 3641 | proxy.url, |
| 3642 | ); |
| 3643 | const trigger = yield* Effect.sync(() => |
| 3644 | Redacted.make( |
| 3645 | Array.from(randomBytes(32), (byte) => byte.toString(16).padStart(2, "0")).join( |
| 3646 | "", |
| 3647 | ), |
| 3648 | ), |
| 3649 | ); |
| 3650 | const deploy = (count: number, floor?: number) => |
| 3651 | actor.deploy( |
| 3652 | Effect.gen(function* () { |
| 3653 | const app = yield* Site; |
| 3654 | const secret = yield* Token; |
| 3655 | const auth = yield* Fly.Secret("WriterTrigger", { |
| 3656 | app, |
| 3657 | name: TRIGGER_SECRET, |
| 3658 | value: trigger, |
| 3659 | }); |
| 3660 | yield* Fly.IpAssignment("Public", { app, type: "shared_v4" }); |
| 3661 | const writer = yield* Writer.pipe(Effect.provide(writerLayer(auth.digest))); |
| 3662 | const consumer = yield* Fly.Machine("Consumer", { |
| 3663 | app, |
| 3664 | region: "iad", |
| 3665 | image: "nginx:alpine", |
| 3666 | count, |
| 3667 | minSecretsVersion: floor, |
| 3668 | env: { SECRET_RESOURCE_NAME: secret.name }, |
| 3669 | init: { |
| 3670 | exec: [ |
| 3671 | "/bin/sh", |
| 3672 | "-c", |
| 3673 | "case \"$ACCEPTANCE_RUNTIME_SECRET\" in *-two) marker=two;; *-three) marker=three;; *) marker=missing;; esac; printf '%s' \"$marker\" > /usr/share/nginx/html/marker; exec nginx -g 'daemon off;'", |
| 3674 | ], |
| 3675 | }, |
| 3676 | checks, |
| 3677 | deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" }, |
| 3678 | shutdown: { signal: "SIGQUIT", timeout: "5 seconds" }, |
| 3679 | }); |
| 3680 | return { app, secret, writer, consumer }; |
| 3681 | }), |
| 3682 | ); |
| 3683 | |
| 3684 | const initial = yield* deploy(1); |
| 3685 | appName = initial.app.appName; |
| 3686 | expect(initial.writer.machineIds).toHaveLength(1); |
| 3687 | expect(initial.consumer.machineIds).toHaveLength(1); |
| 3688 | expect(initial.consumer.imageRef?.digest).toBeTruthy(); |
| 3689 | expect(yield* marker(appName, initial.consumer.machineId)).toEqual({ |
| 3690 | code: 0, |
| 3691 | marker: "two", |
| 3692 | }); |
| 3693 | const before = yield* probeWriter(appName, trigger, "GET").pipe( |
| 3694 | Effect.retry({ |
| 3695 | while: (error) => |
| 3696 | error.stage === "transport" || |
| 3697 | (error.stage === "status" && [404, 502, 503].includes(error.status ?? 0)), |
| 3698 | schedule: Schedule.spaced("2 seconds"), |
| 3699 | times: 8, |
| 3700 | }), |
| 3701 | Effect.timeout("90 seconds"), |
| 3702 | ); |
| 3703 | expect(before).toEqual({ |
| 3704 | version: 0, |
| 3705 | machineId: initial.writer.machineId, |
| 3706 | marker: "ready", |
| 3707 | }); |
| 3708 | const client = yield* HttpClient.HttpClient; |
| 3709 | const unauthorized = yield* client.post(`https://${appName}.fly.dev/writer`).pipe( |
| 3710 | Effect.timeout("30 seconds"), |
| 3711 | Effect.mapError(() => new WriterProbeFailed({ stage: "transport" })), |
| 3712 | ); |
| 3713 | expect(unauthorized.status).toBe(401); |
| 3714 | const initialCensus = yield* census(appName); |
| 3715 | expect(initialCensus.map((machine) => machine.id).sort()).toEqual( |
| 3716 | [...initial.writer.machineIds, ...initial.consumer.machineIds].sort(), |
| 3717 | ); |
| 3718 | const writerBefore = initialCensus.find( |
| 3719 | (machine) => machine.id === initial.writer.machineId, |
| 3720 | ); |
| 3721 | expect(writerBefore?.instance_id).toBeTruthy(); |
| 3722 | expect(writerBefore?.image_ref?.digest).toBeTruthy(); |
| 3723 | const secretWrites = proxy.events.filter( |
| 3724 | (event) => |
| 3725 | event.stage === "completed" && |
| 3726 | event.method === "POST" && |
| 3727 | event.path.startsWith(`/v1/apps/${appName}/secrets`) && |
| 3728 | event.status !== undefined && |
| 3729 | event.status >= 200 && |
| 3730 | event.status < 300 && |
| 3731 | event.secretsVersion !== undefined, |
| 3732 | ); |
| 3733 | expect(secretWrites.length).toBeGreaterThan(0); |
| 3734 | const floor = Math.max(...secretWrites.map((event) => event.secretsVersion!)); |
| 3735 | expect(Number.isSafeInteger(floor)).toBe(true); |
| 3736 | expect(floor).toBeGreaterThan(0); |
| 3737 | const start = proxy.events.length; |
| 3738 | const machinePath = `/v1/apps/${appName}/machines`; |
| 3739 | yield* Effect.sync(() => |
| 3740 | proxy.arm({ |
| 3741 | match: (event) => |
| 3742 | event.method === "POST" && |
| 3743 | event.path === machinePath && |
| 3744 | event.phase === "candidate", |
| 3745 | action: "hold-response", |
| 3746 | remaining: 1, |
| 3747 | }), |
| 3748 | ); |
| 3749 | yield* Effect.gen(function* () { |
| 3750 | const rollout = yield* deploy(2, floor).pipe(Effect.scoped, Effect.forkScoped); |
| 3751 | const held = yield* proxy.wait( |
| 3752 | (event) => |
| 3753 | event.stage === "held" && |
| 3754 | event.path === machinePath && |
| 3755 | event.status !== undefined && |
| 3756 | event.status >= 200 && |
| 3757 | event.status < 300, |
| 3758 | ); |
| 3759 | expect(held.machineId).toBeTruthy(); |
| 3760 | expect(held.minSecretsVersion).toBe(floor); |
| 3761 | // This POST runs the existing WriteSecret binding inside the sibling Machine. |
| 3762 | const later = yield* probeWriter(initial.app.appName, trigger, "POST"); |
| 3763 | expect(later.machineId).toBe(initial.writer.machineId); |
| 3764 | expect(later.marker).toBe("three"); |
| 3765 | expect(Number.isSafeInteger(later.version)).toBe(true); |
| 3766 | expect(later.version).toBeGreaterThan(floor); |
| 3767 | expect( |
| 3768 | proxy.events.some( |
| 3769 | (event) => event.sequence === held.sequence && event.stage === "forwarded", |
| 3770 | ), |
| 3771 | ).toBe(false); |
| 3772 | yield* Effect.sync(proxy.release); |
| 3773 | const next = yield* Fiber.join(rollout).pipe(Effect.timeout("240 seconds")); |
| 3774 | expect(next.app.appName).toBe(initial.app.appName); |
| 3775 | expect(next.secret.name).toBe(initial.secret.name); |
| 3776 | expect(next.writer.machineIds).toEqual(initial.writer.machineIds); |
| 3777 | expect(next.consumer.machineIds).toHaveLength(2); |
| 3778 | expect(next.consumer.machineIds).toContain(held.machineId); |
| 3779 | expect(next.consumer.machineIds).not.toContain(initial.consumer.machineId); |
| 3780 | const events = proxy.events.slice(start); |
| 3781 | const creates = events.filter( |
| 3782 | (event) => |
| 3783 | event.stage === "request" && |
| 3784 | event.method === "POST" && |
| 3785 | event.path === machinePath, |
| 3786 | ); |
| 3787 | expect(creates).toHaveLength(2); |
| 3788 | expect(creates[0]?.sequence).toBe(held.sequence); |
| 3789 | expect(creates.every((event) => event.minSecretsVersion === floor)).toBe(true); |
| 3790 | const released = events.findIndex( |
| 3791 | (event) => event.sequence === held.sequence && event.stage === "forwarded", |
| 3792 | ); |
| 3793 | expect(released).toBeGreaterThan(-1); |
| 3794 | expect(events.indexOf(creates[1]!)).toBeGreaterThan(released); |
| 3795 | expect( |
| 3796 | events.some( |
| 3797 | (event) => |
| 3798 | event.stage === "request" && |
| 3799 | event.machineId === initial.writer.machineId && |
| 3800 | event.method !== "GET", |
| 3801 | ), |
| 3802 | ).toBe(false); |
| 3803 | const live = yield* census(initial.app.appName); |
| 3804 | expect(live.map((machine) => machine.id).sort()).toEqual( |
| 3805 | [...next.writer.machineIds, ...next.consumer.machineIds].sort(), |
| 3806 | ); |
| 3807 | const writerAfter = live.find((machine) => machine.id === initial.writer.machineId); |
| 3808 | expect(writerAfter?.instance_id).toBe(writerBefore?.instance_id); |
| 3809 | expect(writerAfter?.image_ref?.digest).toBe(writerBefore?.image_ref?.digest); |
| 3810 | for (const id of next.consumer.machineIds) { |
| 3811 | const machine = live.find((machine) => machine.id === id); |
| 3812 | expect(machine?.config?.metadata?.["alchemy.phase"]).toBe("active"); |
| 3813 | expect(machine?.image_ref?.digest).toBe(initial.consumer.imageRef?.digest); |
| 3814 | const observed = yield* marker(initial.app.appName, id); |
| 3815 | expect(observed.code).toBe(0); |
| 3816 | expect(id === held.machineId ? ["two", "three"] : ["three"]).toContain( |
| 3817 | observed.marker, |
| 3818 | ); |
| 3819 | } |
| 3820 | expect(yield* probeWriter(initial.app.appName, trigger, "GET")).toEqual(before); |
| 3821 | }).pipe( |
| 3822 | Effect.ensuring( |
| 3823 | Effect.sync(() => { |
| 3824 | proxy.clear(); |
| 3825 | proxy.release(); |
| 3826 | }), |
| 3827 | ), |
| 3828 | Effect.scoped, |
| 3829 | ); |
| 3830 | }).pipe(Effect.scoped), |
| 3831 | { timeout: 900_000 }, |
| 3832 | ); |
| 3833 | }, |
| 3834 | ); |
| 3835 | |
| 3836 | describe.sequential( |
| 3837 | "readiness oracle", |
| 3838 | { tags: ["unit", "provider:fly", "provider:fly:machine", "provider:fly:service", "local"] }, |
| 3839 | () => { |
| 3840 | const names = ["ready", "servicecheck-00-http-80"]; |
| 3841 | const reports = (mirror = false) => |
| 3842 | [...names, ...(mirror ? ["bg_deployments_compat-00-http-80"] : [])].map((name) => ({ |
| 3843 | name, |
| 3844 | status: "passing", |
| 3845 | })); |
| 3846 | |
| 3847 | // Synthetic journals exercise the oracle only; they are not native Fly evidence. |
| 3848 | const trace = (idle = true, retry = false, mirror = false) => { |
| 3849 | const events: ReadinessEvent[] = []; |
| 3850 | const path = (slot: number) => `/v1/apps/oracle/machines/green-${slot}`; |
| 3851 | const metadata = (slot: number, phase: string): Record<string, string> => ({ |
| 3852 | "alchemy.stack": "oracle", |
| 3853 | "alchemy.stage": "pure", |
| 3854 | "alchemy.id": "Worker", |
| 3855 | "alchemy.type": "Fly.Machine", |
| 3856 | "alchemy.instance": "resource-instance", |
| 3857 | "alchemy.fqn": "Worker", |
| 3858 | "alchemy.replica": String(slot), |
| 3859 | "alchemy.generation": "generation", |
| 3860 | "alchemy.workload": "workload", |
| 3861 | "alchemy.image": "nginx@sha256:fixture", |
| 3862 | "alchemy.phase": phase, |
| 3863 | "alchemy.readiness-role": "run", |
| 3864 | "alchemy.readiness-roles": "run,run", |
| 3865 | "alchemy.idle-policy-restored": "true", |
| 3866 | ...(phase === "active" ? { "alchemy.checked-instance": `instance-${slot}` } : {}), |
| 3867 | }); |
| 3868 | const request = (input: Omit<ReadinessEvent, "stage" | "sequence">) => { |
| 3869 | const event: ReadinessEvent = { ...input, stage: "request", sequence: events.length }; |
| 3870 | events.push(event); |
| 3871 | return event; |
| 3872 | }; |
| 3873 | const reply = (input: ReadinessEvent, fields: Partial<ReadinessEvent> = {}) => { |
| 3874 | const event: ReadinessEvent = { ...input, stage: "forwarded", status: 200, ...fields }; |
| 3875 | events.push(event); |
| 3876 | return event; |
| 3877 | }; |
| 3878 | const read = (slot: number, phase: string, state: string) => |
| 3879 | reply(request({ method: "GET", path: path(slot), machineId: `green-${slot}` }), { |
| 3880 | instanceId: `instance-${slot}`, |
| 3881 | digest: "sha256:fixture", |
| 3882 | state, |
| 3883 | cordoned: false, |
| 3884 | phase, |
| 3885 | metadata: metadata(slot, phase), |
| 3886 | checks: state === "started" ? reports(mirror) : [], |
| 3887 | services: [ |
| 3888 | { protocol: "tcp", port: 80, autostop: "stop", autostart: true, floor: 0 }, |
| 3889 | ], |
| 3890 | }); |
| 3891 | const stamp = (slot: number, phase: string) => { |
| 3892 | const input = { |
| 3893 | method: "PATCH", |
| 3894 | path: `${path(slot)}/metadata`, |
| 3895 | machineId: `green-${slot}`, |
| 3896 | phase, |
| 3897 | metadata: metadata(slot, phase), |
| 3898 | metadataOnly: true, |
| 3899 | }; |
| 3900 | if (retry) reply(request(input), { status: phase === "active" ? 503 : 429 }); |
| 3901 | reply(request(input)); |
| 3902 | }; |
| 3903 | for (const slot of [0, 1]) { |
| 3904 | reply(request({ method: "POST", path: path(slot), machineId: `green-${slot}` })); |
| 3905 | read(slot, "promoting", "started"); |
| 3906 | } |
| 3907 | for (const slot of [0, 1]) stamp(slot, "validating"); |
| 3908 | for (const slot of [0, 1]) |
| 3909 | read(slot, "validating", idle && slot === 0 ? "stopped" : "started"); |
| 3910 | for (const slot of [1, 0]) stamp(slot, "active"); |
| 3911 | request({ |
| 3912 | method: "POST", |
| 3913 | path: "/v1/apps/oracle/machines/old/cordon", |
| 3914 | machineId: "old", |
| 3915 | }); |
| 3916 | const candidates: Machine[] = [0, 1].map((slot) => ({ |
| 3917 | id: `green-${slot}`, |
| 3918 | instance_id: `instance-${slot}`, |
| 3919 | image_ref: { digest: "sha256:fixture" }, |
| 3920 | state: idle && slot === 0 ? "stopped" : "started", |
| 3921 | cordoned: false, |
| 3922 | config: { |
| 3923 | metadata: metadata(slot, "active"), |
| 3924 | services: [ |
| 3925 | { |
| 3926 | protocol: "tcp", |
| 3927 | internal_port: 80, |
| 3928 | autostop: "stop", |
| 3929 | autostart: true, |
| 3930 | min_machines_running: 0, |
| 3931 | }, |
| 3932 | ], |
| 3933 | }, |
| 3934 | })); |
| 3935 | return { events, candidates }; |
| 3936 | }; |
| 3937 | |
| 3938 | it.effect( |
| 3939 | "pure readiness trace accepts rejected resume only with fresh started-instance proof", |
| 3940 | () => |
| 3941 | Effect.sync(() => { |
| 3942 | const rejectedStart = (status: number) => { |
| 3943 | const value = trace(false); |
| 3944 | const request: ReadinessEvent = { |
| 3945 | sequence: 999, |
| 3946 | stage: "request", |
| 3947 | method: "POST", |
| 3948 | path: "/v1/apps/oracle/machines/green-0/start", |
| 3949 | machineId: "green-0", |
| 3950 | }; |
| 3951 | value.events.splice(2, 0, request, { ...request, stage: "forwarded", status }); |
| 3952 | return value; |
| 3953 | }; |
| 3954 | expect(() => check(rejectedStart(412))).not.toThrow(); |
| 3955 | expect(() => check(rejectedStart(500))).toThrow(); |
| 3956 | const missing = rejectedStart(412); |
| 3957 | for (const event of missing.events) |
| 3958 | if (event.method === "GET" && event.machineId === "green-0") |
| 3959 | event.state = "stopped"; |
| 3960 | expect(() => check(missing)).toThrow(); |
| 3961 | }), |
| 3962 | ); |
| 3963 | |
| 3964 | type Trace = ReturnType<typeof trace>; |
| 3965 | const check = ({ events, candidates }: Trace, allowIdle = true) => |
| 3966 | assertReadinessCommit(events, ["old"], candidates, [0, 1], names, allowIdle); |
| 3967 | const restored = ({ events }: Trace) => |
| 3968 | events.find( |
| 3969 | (event) => |
| 3970 | event.stage === "forwarded" && |
| 3971 | event.method === "GET" && |
| 3972 | event.machineId === "green-0" && |
| 3973 | event.phase === "promoting", |
| 3974 | )!; |
| 3975 | const pending = ({ events }: Trace) => |
| 3976 | events.find( |
| 3977 | (event) => |
| 3978 | event.stage === "forwarded" && |
| 3979 | event.method === "GET" && |
| 3980 | event.machineId === "green-0" && |
| 3981 | event.phase === "validating", |
| 3982 | )!; |
| 3983 | const active = ({ events }: Trace) => |
| 3984 | events.filter( |
| 3985 | (event) => |
| 3986 | event.method === "PATCH" && event.machineId === "green-0" && event.phase === "active", |
| 3987 | ); |
| 3988 | |
| 3989 | it.effect("pure readiness matcher permits only passing corresponding mirrors", () => |
| 3990 | Effect.sync(() => { |
| 3991 | expect(readinessChecksPassing(reports(), names)).toBe(true); |
| 3992 | expect(readinessChecksPassing(reports(true), names)).toBe(true); |
| 3993 | for (const invalid of [ |
| 3994 | undefined, |
| 3995 | [], |
| 3996 | reports(true).filter((check) => check.name !== "servicecheck-00-http-80"), |
| 3997 | [...reports(), reports()[0]!], |
| 3998 | [...reports(true), reports(true).at(-1)!], |
| 3999 | [...reports(), { name: "unrelated", status: "passing" }], |
| 4000 | [...reports(), { name: "bg_deployments_compat-00-tcp-80", status: "passing" }], |
| 4001 | [...reports(), { name: undefined, status: "passing" }], |
| 4002 | reports(true).map((check) => ({ ...check, status: "warning" })), |
| 4003 | reports(true).map((check) => |
| 4004 | check.name.startsWith("bg_") ? { ...check, status: "critical" } : check, |
| 4005 | ), |
| 4006 | ]) |
| 4007 | expect(readinessChecksPassing(invalid, names)).toBe(false); |
| 4008 | expect(readinessChecksPassing(reports(), [...names, names[0]!])).toBe(false); |
| 4009 | }), |
| 4010 | ); |
| 4011 | |
| 4012 | it.effect( |
| 4013 | "pure readiness traces accept running and restored-idle proof with settled metadata retries", |
| 4014 | () => |
| 4015 | Effect.sync(() => { |
| 4016 | for (const idle of [false, true]) { |
| 4017 | for (const retry of [false, true]) check(trace(idle, retry, true)); |
| 4018 | } |
| 4019 | const suspended = trace(); |
| 4020 | pending(suspended).state = "suspended"; |
| 4021 | suspended.candidates[0]!.state = "suspended"; |
| 4022 | check(suspended); |
| 4023 | }), |
| 4024 | ); |
| 4025 | |
| 4026 | const invalidTraces: [string, (value: Trace) => void][] = [ |
| 4027 | [ |
| 4028 | "missing restored passing proof", |
| 4029 | (value) => { |
| 4030 | restored(value).checks = []; |
| 4031 | }, |
| 4032 | ], |
| 4033 | [ |
| 4034 | "proof from another instance", |
| 4035 | (value) => { |
| 4036 | restored(value).instanceId = "obsolete"; |
| 4037 | }, |
| 4038 | ], |
| 4039 | [ |
| 4040 | "proof from another workload", |
| 4041 | (value) => { |
| 4042 | restored(value).metadata!["alchemy.workload"] = "obsolete"; |
| 4043 | }, |
| 4044 | ], |
| 4045 | [ |
| 4046 | "proof with the preparation service policy", |
| 4047 | (value) => { |
| 4048 | restored(value).services![0]!.autostop = "off"; |
| 4049 | }, |
| 4050 | ], |
| 4051 | [ |
| 4052 | "created representative", |
| 4053 | (value) => { |
| 4054 | pending(value).state = "created"; |
| 4055 | }, |
| 4056 | ], |
| 4057 | [ |
| 4058 | "configuration mutation after restored proof", |
| 4059 | (value) => { |
| 4060 | const index = value.events.indexOf(restored(value)) + 1; |
| 4061 | const event: ReadinessEvent = { |
| 4062 | sequence: 10_000, |
| 4063 | stage: "request", |
| 4064 | method: "POST", |
| 4065 | path: "/v1/apps/oracle/machines/green-0", |
| 4066 | machineId: "green-0", |
| 4067 | }; |
| 4068 | value.events.splice(index, 0, event, { ...event, stage: "forwarded", status: 200 }); |
| 4069 | }, |
| 4070 | ], |
| 4071 | [ |
| 4072 | "GET requested before restoration completed", |
| 4073 | (value) => { |
| 4074 | const response = restored(value); |
| 4075 | const index = value.events.findIndex((event) => event.sequence === response.sequence); |
| 4076 | const [request] = value.events.splice(index, 1); |
| 4077 | value.events.splice(1, 0, request!); |
| 4078 | }, |
| 4079 | ], |
| 4080 | [ |
| 4081 | "changed pending metadata after proof", |
| 4082 | (value) => { |
| 4083 | for (const event of value.events.filter( |
| 4084 | (event) => |
| 4085 | event.method === "PATCH" && |
| 4086 | event.machineId === "green-0" && |
| 4087 | event.phase === "validating", |
| 4088 | )) { |
| 4089 | event.metadata = { ...event.metadata, "alchemy.checked-instance": "unproven" }; |
| 4090 | } |
| 4091 | }, |
| 4092 | ], |
| 4093 | [ |
| 4094 | "conflicting commit retry", |
| 4095 | (value) => { |
| 4096 | const retry = active(value) |
| 4097 | .filter((event) => event.stage === "request") |
| 4098 | .at(-1)!; |
| 4099 | retry.metadata = { ...retry.metadata, "alchemy.checked-instance": "other" }; |
| 4100 | }, |
| 4101 | ], |
| 4102 | [ |
| 4103 | "unsettled rejected attempt", |
| 4104 | (value) => { |
| 4105 | value.events.splice(value.events.indexOf(active(value)[1]!), 1); |
| 4106 | }, |
| 4107 | ], |
| 4108 | [ |
| 4109 | "retry begins before rejected receipt", |
| 4110 | (value) => { |
| 4111 | const attempts = active(value); |
| 4112 | const index = value.events.indexOf(attempts[2]!); |
| 4113 | value.events.splice(index, 1); |
| 4114 | value.events.splice(value.events.indexOf(attempts[1]!), 0, attempts[2]!); |
| 4115 | }, |
| 4116 | ], |
| 4117 | [ |
| 4118 | "terminal commit rejection", |
| 4119 | (value) => { |
| 4120 | active(value).at(-1)!.status = 429; |
| 4121 | }, |
| 4122 | ], |
| 4123 | [ |
| 4124 | "nonretryable metadata rejection", |
| 4125 | (value) => { |
| 4126 | active(value)[1]!.status = 403; |
| 4127 | }, |
| 4128 | ], |
| 4129 | [ |
| 4130 | "duplicate successful commit", |
| 4131 | (value) => { |
| 4132 | active(value)[1]!.status = 200; |
| 4133 | }, |
| 4134 | ], |
| 4135 | [ |
| 4136 | "replica zero starts committing before the other commit settles", |
| 4137 | (value) => { |
| 4138 | const response = value.events.find( |
| 4139 | (event) => |
| 4140 | event.machineId === "green-1" && |
| 4141 | event.phase === "active" && |
| 4142 | event.stage === "forwarded" && |
| 4143 | event.status === 200, |
| 4144 | )!; |
| 4145 | value.events.splice(value.events.indexOf(response), 1); |
| 4146 | value.events.splice(value.events.indexOf(active(value)[0]!) + 1, 0, response); |
| 4147 | }, |
| 4148 | ], |
| 4149 | [ |
| 4150 | "retirement before terminal commit receipt", |
| 4151 | (value) => { |
| 4152 | const retirement = value.events.pop()!; |
| 4153 | value.events.splice(value.events.indexOf(active(value).at(-1)!), 0, retirement); |
| 4154 | }, |
| 4155 | ], |
| 4156 | [ |
| 4157 | "commit before complete topology validation", |
| 4158 | (value) => { |
| 4159 | const first = value.events.findIndex((event) => event.phase === "active"); |
| 4160 | const commit = value.events.splice(first, 4); |
| 4161 | const validation = value.events.findIndex( |
| 4162 | (event) => |
| 4163 | event.method === "GET" && |
| 4164 | event.machineId === "green-1" && |
| 4165 | event.stage === "forwarded" && |
| 4166 | event.phase === "validating", |
| 4167 | ); |
| 4168 | value.events.splice(validation - 1, 0, ...commit); |
| 4169 | }, |
| 4170 | ], |
| 4171 | ]; |
| 4172 | |
| 4173 | for (const [name, corrupt] of invalidTraces) { |
| 4174 | it.effect(`pure readiness trace rejects ${name}`, () => |
| 4175 | Effect.sync(() => { |
| 4176 | const value = trace(true, true); |
| 4177 | corrupt(value); |
| 4178 | expect(() => check(value)).toThrow(); |
| 4179 | }), |
| 4180 | ); |
| 4181 | } |
| 4182 | |
| 4183 | it.effect("pure readiness trace refuses idle completion when autostop is disabled", () => |
| 4184 | Effect.sync(() => { |
| 4185 | expect(() => check(trace(), false)).toThrow(); |
| 4186 | }), |
| 4187 | ); |
| 4188 | }, |
| 4189 | ); |
| 4190 | |
| 4191 | describe.sequential( |
| 4192 | "engine state", |
| 4193 | { |
| 4194 | tags: [ |
| 4195 | "provider:fly", |
| 4196 | "provider:fly:app", |
| 4197 | "provider:fly:machine", |
| 4198 | "provider:fly:service", |
| 4199 | "live", |
| 4200 | ], |
| 4201 | }, |
| 4202 | () => { |
| 4203 | const file = "test/Fly/BlueGreen.test.ts"; |
| 4204 | |
| 4205 | const { test } = Test.make({ providers: Fly.providers() }); |
| 4206 | |
| 4207 | test.provider( |
| 4208 | "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App", |
| 4209 | (stack) => |
| 4210 | Effect.gen(function* () { |
| 4211 | yield* stack.destroy(); |
| 4212 | const proxy = yield* transportProxy(); |
| 4213 | const firstActor = yield* engineActor( |
| 4214 | stack, |
| 4215 | "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App", |
| 4216 | file, |
| 4217 | proxy.url, |
| 4218 | ); |
| 4219 | const first = yield* firstActor.deploy(Fly.App("Site")).pipe(Effect.scoped); |
| 4220 | const secondActor = yield* engineActor( |
| 4221 | stack, |
| 4222 | "F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App", |
| 4223 | file, |
| 4224 | proxy.url, |
| 4225 | ); |
| 4226 | expect(secondActor.state).not.toBe(firstActor.state); |
| 4227 | const second = yield* secondActor.deploy(Fly.App("Site")).pipe(Effect.scoped); |
| 4228 | expect(second.appName).toBe(first.appName); |
| 4229 | expect(second.appId).toBe(first.appId); |
| 4230 | expect( |
| 4231 | proxy.events.filter( |
| 4232 | (event) => |
| 4233 | event.stage === "completed" && |
| 4234 | event.method === "POST" && |
| 4235 | event.path === "/v1/apps" && |
| 4236 | event.status! < 300, |
| 4237 | ), |
| 4238 | ).toHaveLength(1); |
| 4239 | yield* stack.destroy(); |
| 4240 | yield* assertAppGone(first.appName); |
| 4241 | }).pipe(Effect.scoped), |
| 4242 | { timeout: 180_000 }, |
| 4243 | ); |
| 4244 | }, |
| 4245 | ); |
| 4246 | }); |
| 4247 | describe.sequential("shard 4", () => { |
| 4248 | describe.sequential( |
| 4249 | "check cadence", |
| 4250 | { |
| 4251 | tags: [ |
| 4252 | "provider:fly", |
| 4253 | "provider:fly:app", |
| 4254 | "provider:fly:machine", |
| 4255 | "provider:fly:service", |
| 4256 | "live", |
| 4257 | ], |
| 4258 | }, |
| 4259 | () => { |
| 4260 | const file = "test/Fly/BlueGreen.test.ts"; |
| 4261 | const { test } = Test.make({ providers: Fly.providers() }); |
| 4262 | |
| 4263 | const cadenceCheck = { |
| 4264 | ...checks.ready, |
| 4265 | path: "/cadence-ready", |
| 4266 | interval: "75s", |
| 4267 | gracePeriod: "0s", |
| 4268 | }; |
| 4269 | |
| 4270 | describe.sequential("live long-cadence readiness and service interval limits", () => { |
| 4271 | for (const kind of ["named", "service"] as const) { |
| 4272 | for (const sufficient of [false, true]) { |
| 4273 | const budget = sufficient ? 120_000 : 35_000; |
| 4274 | |
| 4275 | test.provider( |
| 4276 | `S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`, |
| 4277 | (stack) => |
| 4278 | Effect.gen(function* () { |
| 4279 | yield* stack.destroy(); |
| 4280 | const site = yield* stack.deploy(Fly.App("Site")); |
| 4281 | try { |
| 4282 | const first = yield* deployWorker(stack, "one"); |
| 4283 | const proxy = yield* readinessProxy(); |
| 4284 | const actor = yield* readinessActor( |
| 4285 | stack, |
| 4286 | `S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`, |
| 4287 | file, |
| 4288 | proxy, |
| 4289 | ); |
| 4290 | const props: Partial<Omit<Extract<MachineProps, { image: string }>, "app">> = |
| 4291 | { |
| 4292 | init: { |
| 4293 | exec: [ |
| 4294 | "/bin/sh", |
| 4295 | "-c", |
| 4296 | "rm -f /usr/share/nginx/html/cadence-ready; (sleep 80; touch /usr/share/nginx/html/cadence-ready) & exec nginx -g 'daemon off;'", |
| 4297 | ], |
| 4298 | }, |
| 4299 | checks: kind === "named" ? { ready: cadenceCheck } : {}, |
| 4300 | services: |
| 4301 | kind === "service" |
| 4302 | ? [ |
| 4303 | { |
| 4304 | protocol: "tcp", |
| 4305 | internalPort: 80, |
| 4306 | autostop: "off", |
| 4307 | checks: [cadenceCheck], |
| 4308 | }, |
| 4309 | ] |
| 4310 | : [], |
| 4311 | deploy: { strategy: "bluegreen", healthTimeout: budget }, |
| 4312 | }; |
| 4313 | yield* Effect.gen(function* () { |
| 4314 | const started = yield* Clock.currentTimeMillis; |
| 4315 | const attempt = yield* deployWorker(actor, "two", props).pipe( |
| 4316 | Effect.scoped, |
| 4317 | Effect.result, |
| 4318 | Effect.forkScoped, |
| 4319 | ); |
| 4320 | const checkName = kind === "named" ? "ready" : "servicecheck-00-http-80"; |
| 4321 | const failed = yield* census(site.appName).pipe( |
| 4322 | Effect.map((live) => |
| 4323 | live.find( |
| 4324 | (machine) => |
| 4325 | !first.machineIds.includes(machine.id!) && |
| 4326 | machine.state === "started" && |
| 4327 | machine.checks?.some( |
| 4328 | (check) => |
| 4329 | check.name === checkName && check.status === "critical", |
| 4330 | ), |
| 4331 | ), |
| 4332 | ), |
| 4333 | Effect.repeat({ |
| 4334 | schedule: Schedule.spaced("1 second"), |
| 4335 | times: 120, |
| 4336 | until: (machine) => machine !== undefined, |
| 4337 | }), |
| 4338 | Effect.timeout("150 seconds"), |
| 4339 | Effect.raceFirst( |
| 4340 | Effect.gen(function* () { |
| 4341 | const result = yield* Fiber.join(attempt); |
| 4342 | if (Result.isFailure(result)) |
| 4343 | return yield* Effect.fail(result.failure); |
| 4344 | return yield* Effect.fail( |
| 4345 | new Error( |
| 4346 | "Candidate committed before a failing long-cadence report was observed", |
| 4347 | ), |
| 4348 | ); |
| 4349 | }), |
| 4350 | ), |
| 4351 | ); |
| 4352 | expect(failed).toBeDefined(); |
| 4353 | if (!failed) |
| 4354 | return yield* Effect.fail( |
| 4355 | new Error("Fly never emitted the initial failing cadence report"), |
| 4356 | ); |
| 4357 | const nativeCheck = |
| 4358 | kind === "named" |
| 4359 | ? failed.config?.checks?.ready |
| 4360 | : failed.config?.services?.[0]?.checks?.[0]; |
| 4361 | expect( |
| 4362 | kind === "named" ? ["75s", "1m15s"] : ["60s", "1m0s", "1m"], |
| 4363 | ).toContain(nativeCheck?.interval); |
| 4364 | expect(["0s", "0", undefined]).toContain(nativeCheck?.grace_period); |
| 4365 | expect(failed.cordoned).toBe(true); |
| 4366 | const failureReport = failed.checks!.find( |
| 4367 | (check) => check.name === checkName, |
| 4368 | )!; |
| 4369 | expect(failureReport.updated_at).toBeDefined(); |
| 4370 | if (sufficient) { |
| 4371 | const passing = yield* machines |
| 4372 | .getMachine({ app_name: site.appName, machine_id: failed.id! }) |
| 4373 | .pipe( |
| 4374 | Effect.repeat({ |
| 4375 | schedule: Schedule.spaced("1 second"), |
| 4376 | times: 120, |
| 4377 | until: (machine) => |
| 4378 | machine.checks?.some( |
| 4379 | (check) => |
| 4380 | check.name === checkName && check.status === "passing", |
| 4381 | ) === true, |
| 4382 | }), |
| 4383 | Effect.timeout("120 seconds"), |
| 4384 | ); |
| 4385 | const report = passing.checks?.find((check) => check.name === checkName); |
| 4386 | expect(report?.status).toBe("passing"); |
| 4387 | expect(report?.updated_at).toBeDefined(); |
| 4388 | expect(failed.instance_id).toBeDefined(); |
| 4389 | expect(passing.instance_id).toBe(failed.instance_id); |
| 4390 | const reportSpacing = yield* Effect.sync( |
| 4391 | () => |
| 4392 | Date.parse(report!.updated_at!) - |
| 4393 | Date.parse(failureReport.updated_at!), |
| 4394 | ); |
| 4395 | expect(reportSpacing).toBeGreaterThan(60_000); |
| 4396 | yield* Effect.logInfo( |
| 4397 | "Native cadence reports across autonomous readiness transition", |
| 4398 | { |
| 4399 | reportSpacingMs: reportSpacing, |
| 4400 | failingReport: failureReport.updated_at, |
| 4401 | passingReport: report!.updated_at, |
| 4402 | }, |
| 4403 | ); |
| 4404 | const result = yield* Fiber.join(attempt).pipe( |
| 4405 | Effect.timeout("600 seconds"), |
| 4406 | ); |
| 4407 | expect(Result.isSuccess(result)).toBe(true); |
| 4408 | if (Result.isFailure(result)) return yield* Effect.fail(result.failure); |
| 4409 | expect(result.success.machineIds).toEqual([failed.id]); |
| 4410 | const committed = yield* assertCommitted( |
| 4411 | site.appName, |
| 4412 | result.success.machineIds, |
| 4413 | ); |
| 4414 | expect(committed[0]!.instance_id).toBe(failed.instance_id); |
| 4415 | assertReadinessCommit( |
| 4416 | proxy.readiness, |
| 4417 | first.machineIds, |
| 4418 | committed, |
| 4419 | [0], |
| 4420 | [checkName], |
| 4421 | false, |
| 4422 | ); |
| 4423 | } else { |
| 4424 | const result = yield* Fiber.join(attempt).pipe( |
| 4425 | Effect.timeout("180 seconds"), |
| 4426 | ); |
| 4427 | expect(Result.isFailure(result)).toBe(true); |
| 4428 | if (Result.isFailure(result)) |
| 4429 | expect(result.failure._tag).toBe("Fly.ReplicaChecksNotPassing"); |
| 4430 | expect((yield* Clock.currentTimeMillis) - started).toBeLessThan(180_000); |
| 4431 | const live = yield* census(site.appName); |
| 4432 | expect(live.map((machine) => machine.id)).toEqual(first.machineIds); |
| 4433 | expect(live[0]!.state).toBe("started"); |
| 4434 | expect(live[0]!.cordoned).toBe(false); |
| 4435 | expect( |
| 4436 | proxy.events.some( |
| 4437 | (event) => |
| 4438 | event.stage === "request" && |
| 4439 | first.machineIds.includes(event.machineId!) && |
| 4440 | (event.path.endsWith("/stop") || |
| 4441 | event.path.endsWith("/cordon") || |
| 4442 | event.path.endsWith("/suspend") || |
| 4443 | (event.path.endsWith("/metadata") && |
| 4444 | event.phase === "retiring") || |
| 4445 | (event.method === "DELETE" && |
| 4446 | /\/machines\/[^/]+$/.test(event.path))), |
| 4447 | ), |
| 4448 | ).toBe(false); |
| 4449 | expect( |
| 4450 | proxy.events.some( |
| 4451 | (event) => |
| 4452 | event.machineId === failed.id && event.path.endsWith("/uncordon"), |
| 4453 | ), |
| 4454 | ).toBe(false); |
| 4455 | expect( |
| 4456 | proxy.readiness.some( |
| 4457 | (event) => |
| 4458 | event.stage === "request" && |
| 4459 | event.machineId === failed.id && |
| 4460 | event.path.endsWith("/metadata") && |
| 4461 | event.phase === "active", |
| 4462 | ), |
| 4463 | ).toBe(false); |
| 4464 | } |
| 4465 | yield* Effect.logInfo("Live long-cadence evidence", { |
| 4466 | kind, |
| 4467 | interval: nativeCheck?.interval, |
| 4468 | grace: nativeCheck?.grace_period, |
| 4469 | budget, |
| 4470 | firstFailure: failureReport.updated_at, |
| 4471 | elapsedMs: (yield* Clock.currentTimeMillis) - started, |
| 4472 | }); |
| 4473 | }).pipe(Effect.scoped); |
| 4474 | } finally { |
| 4475 | yield* stack.destroy(); |
| 4476 | yield* assertAppGone(site.appName); |
| 4477 | } |
| 4478 | }).pipe(Effect.scoped), |
| 4479 | { timeout: 1_200_000 }, |
| 4480 | ); |
| 4481 | } |
| 4482 | } |
| 4483 | }); |
| 4484 | }, |
| 4485 | ); |
| 4486 | |
| 4487 | describe.sequential( |
| 4488 | "multi-container", |
| 4489 | { tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"] }, |
| 4490 | () => { |
| 4491 | const { test } = Test.make({ providers: Fly.providers() }); |
| 4492 | const firstImage = |
| 4493 | "docker-hub-mirror.fly.io/library/nginx@sha256:7396be67b6f53012a5cf955fa9040619294c25ccacf11e22af5de1b572fc756e"; |
| 4494 | const nextImage = |
| 4495 | "docker-hub-mirror.fly.io/library/nginx@sha256:ef8676b33d681f272ba429b27658bdd7e640963279714c96bddf1dc76307f7b6"; |
| 4496 | |
| 4497 | interface GroupOptions { |
| 4498 | count?: number; |
| 4499 | /** Reverse declaration order and add empty optional fields. */ |
| 4500 | reordered?: boolean; |
| 4501 | /** Add a third container to the group. */ |
| 4502 | extra?: boolean; |
| 4503 | badHealth?: boolean; |
| 4504 | strategy?: "rolling" | "bluegreen"; |
| 4505 | } |
| 4506 | |
| 4507 | const containers = ( |
| 4508 | sidecarImage: string, |
| 4509 | { reordered = false, extra = false }: GroupOptions, |
| 4510 | ): MachineContainer[] => { |
| 4511 | const group: MachineContainer[] = [ |
| 4512 | { |
| 4513 | name: "web", |
| 4514 | image: firstImage, |
| 4515 | healthChecks: [ |
| 4516 | { name: "web-tcp", kind: "readiness", tcp: { port: 80 }, interval: 5, timeout: 2 }, |
| 4517 | ], |
| 4518 | }, |
| 4519 | { |
| 4520 | name: "sidecar", |
| 4521 | image: sidecarImage, |
| 4522 | cmd: [ |
| 4523 | "sh", |
| 4524 | "-c", |
| 4525 | "sed -i 's/80/8080/g' /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'", |
| 4526 | ], |
| 4527 | dependsOn: [{ name: "web", condition: "healthy" }], |
| 4528 | healthChecks: [ |
| 4529 | { |
| 4530 | name: "sidecar-tcp", |
| 4531 | kind: "readiness", |
| 4532 | tcp: { port: 8080 }, |
| 4533 | interval: 5, |
| 4534 | timeout: 2, |
| 4535 | }, |
| 4536 | ], |
| 4537 | }, |
| 4538 | ...(extra |
| 4539 | ? [{ name: "extra", image: firstImage, cmd: ["sh", "-c", "sleep infinity"] }] |
| 4540 | : []), |
| 4541 | ]; |
| 4542 | return reordered |
| 4543 | ? group.reverse().map((container) => ({ ...container, env: {} })) |
| 4544 | : group; |
| 4545 | }; |
| 4546 | |
| 4547 | const machineChecks = (badHealth: boolean) => ({ |
| 4548 | web: { type: "tcp" as const, port: 80, interval: "2s", timeout: "1s" }, |
| 4549 | sidecar: { |
| 4550 | type: "tcp" as const, |
| 4551 | port: badHealth ? 9999 : 8080, |
| 4552 | interval: "2s", |
| 4553 | timeout: "1s", |
| 4554 | }, |
| 4555 | }); |
| 4556 | |
| 4557 | const deployGroup = ( |
| 4558 | stack: ScratchStack, |
| 4559 | sidecarImage: string, |
| 4560 | options: GroupOptions = {}, |
| 4561 | ) => |
| 4562 | stack.deploy( |
| 4563 | Effect.gen(function* () { |
| 4564 | const app = yield* Fly.App("MultiContainerBlueGreenSite"); |
| 4565 | return yield* Fly.Machine("Group", { |
| 4566 | app, |
| 4567 | region: "fra", |
| 4568 | count: options.count ?? 1, |
| 4569 | guest: { cpus: 1, memoryMb: 256 }, |
| 4570 | containers: containers(sidecarImage, options), |
| 4571 | checks: machineChecks(options.badHealth ?? false), |
| 4572 | deploy: { |
| 4573 | strategy: options.strategy ?? "bluegreen", |
| 4574 | healthTimeout: options.badHealth ? "15 seconds" : "30 seconds", |
| 4575 | }, |
| 4576 | shutdown: { signal: "SIGTERM", timeout: "1 second" }, |
| 4577 | }); |
| 4578 | }), |
| 4579 | ); |
| 4580 | |
| 4581 | const images = (machine: Machine) => |
| 4582 | (machine.config?.containers ?? []) |
| 4583 | .map(({ name, image }) => ({ name, image })) |
| 4584 | .sort((a, b) => (a.name ?? "").localeCompare(b.name ?? "")); |
| 4585 | |
| 4586 | const appGone = (appName: string) => |
| 4587 | machines.getApp({ app_name: appName }).pipe( |
| 4588 | Effect.as(false), |
| 4589 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 4590 | ); |
| 4591 | |
| 4592 | const machineGone = (appName: string, machineId: string) => |
| 4593 | machines.getMachine({ app_name: appName, machine_id: machineId }).pipe( |
| 4594 | Effect.map((machine) => machine.state === "destroyed"), |
| 4595 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 4596 | Effect.repeat({ |
| 4597 | schedule: Schedule.spaced("2 seconds"), |
| 4598 | times: 10, |
| 4599 | until: (gone) => gone, |
| 4600 | }), |
| 4601 | ); |
| 4602 | |
| 4603 | test.provider( |
| 4604 | "checks two named-container replicas and ignores declaration order and empty fields", |
| 4605 | (stack) => |
| 4606 | Effect.gen(function* () { |
| 4607 | yield* stack.destroy(); |
| 4608 | const first = yield* deployGroup(stack, firstImage, { count: 2 }); |
| 4609 | expect(first.machineIds).toHaveLength(2); |
| 4610 | const observed = yield* Effect.forEach(first.machineIds, (id) => |
| 4611 | machines.getMachine({ app_name: first.appName, machine_id: id }), |
| 4612 | ); |
| 4613 | for (const machine of observed) { |
| 4614 | expect(images(machine)).toEqual([ |
| 4615 | { name: "sidecar", image: firstImage }, |
| 4616 | { name: "web", image: firstImage }, |
| 4617 | ]); |
| 4618 | expect(machine.cordoned).toBe(false); |
| 4619 | expect(machine.config?.metadata?.[keys.phase]).toBe("active"); |
| 4620 | expect(machine.config?.metadata?.[keys.protocol]).toBe("2"); |
| 4621 | const ready = yield* waitHealthy(first.appName, machine, 30_000); |
| 4622 | for (const check of ["web", "sidecar"]) |
| 4623 | expect( |
| 4624 | ready.checks?.some( |
| 4625 | ({ name, status }) => name === check && status === "passing", |
| 4626 | ), |
| 4627 | ).toBe(true); |
| 4628 | } |
| 4629 | const reordered = yield* deployGroup(stack, firstImage, { |
| 4630 | count: 2, |
| 4631 | reordered: true, |
| 4632 | }); |
| 4633 | expect([...reordered.machineIds].sort()).toEqual([...first.machineIds].sort()); |
| 4634 | for (const machine of observed) { |
| 4635 | const after = yield* machines.getMachine({ |
| 4636 | app_name: first.appName, |
| 4637 | machine_id: machine.id!, |
| 4638 | }); |
| 4639 | expect(after.instance_id).toBe(machine.instance_id); |
| 4640 | } |
| 4641 | yield* stack.destroy(); |
| 4642 | expect(yield* appGone(first.appName)).toBe(true); |
| 4643 | }), |
| 4644 | { timeout: 180_000 }, |
| 4645 | ); |
| 4646 | |
| 4647 | test.provider( |
| 4648 | "named group transitions from rolling to blue/green and back", |
| 4649 | (stack) => |
| 4650 | Effect.gen(function* () { |
| 4651 | yield* stack.destroy(); |
| 4652 | const rolling = yield* deployGroup(stack, firstImage, { strategy: "rolling" }); |
| 4653 | const promoted = yield* deployGroup(stack, firstImage); |
| 4654 | expect(promoted.machineId).not.toBe(rolling.machineId); |
| 4655 | const promotedMachine = yield* machines.getMachine({ |
| 4656 | app_name: promoted.appName, |
| 4657 | machine_id: promoted.machineId, |
| 4658 | }); |
| 4659 | expect(promotedMachine.config?.metadata?.[keys.protocol]).toBe("2"); |
| 4660 | expect(images(promotedMachine)).toEqual([ |
| 4661 | { name: "sidecar", image: firstImage }, |
| 4662 | { name: "web", image: firstImage }, |
| 4663 | ]); |
| 4664 | const optedOut = yield* deployGroup(stack, firstImage, { strategy: "rolling" }); |
| 4665 | expect(optedOut.machineId).toBe(promoted.machineId); |
| 4666 | const live = yield* machines.getMachine({ |
| 4667 | app_name: optedOut.appName, |
| 4668 | machine_id: optedOut.machineId, |
| 4669 | }); |
| 4670 | expect(live.config?.metadata?.[keys.protocol]).toBeUndefined(); |
| 4671 | expect(images(live)).toEqual([ |
| 4672 | { name: "sidecar", image: firstImage }, |
| 4673 | { name: "web", image: firstImage }, |
| 4674 | ]); |
| 4675 | yield* stack.destroy(); |
| 4676 | expect(yield* appGone(rolling.appName)).toBe(true); |
| 4677 | }), |
| 4678 | { timeout: 180_000 }, |
| 4679 | ); |
| 4680 | |
| 4681 | test.provider( |
| 4682 | "a single-image blue/green generation is replaced by a named group", |
| 4683 | (stack) => |
| 4684 | Effect.gen(function* () { |
| 4685 | yield* stack.destroy(); |
| 4686 | const single = yield* stack.deploy( |
| 4687 | Effect.gen(function* () { |
| 4688 | const app = yield* Fly.App("MultiContainerBlueGreenSite"); |
| 4689 | return yield* Fly.Machine("Group", { |
| 4690 | app, |
| 4691 | region: "fra", |
| 4692 | guest: { cpus: 1, memoryMb: 256 }, |
| 4693 | image: firstImage, |
| 4694 | checks: { web: machineChecks(false).web }, |
| 4695 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 4696 | shutdown: { signal: "SIGTERM", timeout: "1 second" }, |
| 4697 | }); |
| 4698 | }), |
| 4699 | ); |
| 4700 | const before = yield* machines.getMachine({ |
| 4701 | app_name: single.appName, |
| 4702 | machine_id: single.machineId, |
| 4703 | }); |
| 4704 | expect(before.config?.metadata?.[keys.protocol]).toBe("1"); |
| 4705 | const group = yield* deployGroup(stack, firstImage); |
| 4706 | expect(group.machineId).not.toBe(single.machineId); |
| 4707 | const after = yield* machines.getMachine({ |
| 4708 | app_name: group.appName, |
| 4709 | machine_id: group.machineId, |
| 4710 | }); |
| 4711 | expect(after.config?.metadata?.[keys.protocol]).toBe("2"); |
| 4712 | expect(after.config?.metadata?.[keys.phase]).toBe("active"); |
| 4713 | expect(images(after)).toEqual([ |
| 4714 | { name: "sidecar", image: firstImage }, |
| 4715 | { name: "web", image: firstImage }, |
| 4716 | ]); |
| 4717 | expect(yield* machineGone(single.appName, single.machineId)).toBe(true); |
| 4718 | yield* stack.destroy(); |
| 4719 | expect(yield* appGone(single.appName)).toBe(true); |
| 4720 | }), |
| 4721 | { timeout: 180_000 }, |
| 4722 | ); |
| 4723 | |
| 4724 | test.provider( |
| 4725 | "changing one image or adding a container replaces the whole group", |
| 4726 | (stack) => |
| 4727 | Effect.gen(function* () { |
| 4728 | yield* stack.destroy(); |
| 4729 | const first = yield* deployGroup(stack, firstImage); |
| 4730 | const updated = yield* deployGroup(stack, nextImage, { reordered: true }); |
| 4731 | expect(updated.machineId).not.toBe(first.machineId); |
| 4732 | const replacement = yield* machines.getMachine({ |
| 4733 | app_name: updated.appName, |
| 4734 | machine_id: updated.machineId, |
| 4735 | }); |
| 4736 | expect(images(replacement)).toEqual([ |
| 4737 | { name: "sidecar", image: nextImage }, |
| 4738 | { name: "web", image: firstImage }, |
| 4739 | ]); |
| 4740 | expect(replacement.cordoned).toBe(false); |
| 4741 | expect(replacement.config?.metadata?.[keys.phase]).toBe("active"); |
| 4742 | expect(yield* machineGone(first.appName, first.machineId)).toBe(true); |
| 4743 | |
| 4744 | const extended = yield* deployGroup(stack, nextImage, { extra: true }); |
| 4745 | expect(extended.machineId).not.toBe(updated.machineId); |
| 4746 | expect( |
| 4747 | images( |
| 4748 | yield* machines.getMachine({ |
| 4749 | app_name: extended.appName, |
| 4750 | machine_id: extended.machineId, |
| 4751 | }), |
| 4752 | ), |
| 4753 | ).toEqual([ |
| 4754 | { name: "extra", image: firstImage }, |
| 4755 | { name: "sidecar", image: nextImage }, |
| 4756 | { name: "web", image: firstImage }, |
| 4757 | ]); |
| 4758 | expect(yield* machineGone(updated.appName, updated.machineId)).toBe(true); |
| 4759 | yield* stack.destroy(); |
| 4760 | expect(yield* appGone(first.appName)).toBe(true); |
| 4761 | }), |
| 4762 | { timeout: 240_000 }, |
| 4763 | ); |
| 4764 | |
| 4765 | test.provider( |
| 4766 | "bad secondary readiness preserves the serving predecessor", |
| 4767 | (stack) => |
| 4768 | Effect.gen(function* () { |
| 4769 | yield* stack.destroy(); |
| 4770 | const first = yield* deployGroup(stack, firstImage); |
| 4771 | const failed = yield* deployGroup(stack, nextImage, { badHealth: true }).pipe( |
| 4772 | Effect.result, |
| 4773 | ); |
| 4774 | expect(Result.isFailure(failed)).toBe(true); |
| 4775 | const predecessor = yield* machines.getMachine({ |
| 4776 | app_name: first.appName, |
| 4777 | machine_id: first.machineId, |
| 4778 | }); |
| 4779 | expect(predecessor.state).toBe("started"); |
| 4780 | expect(predecessor.cordoned).toBe(false); |
| 4781 | expect(predecessor.config?.metadata?.[keys.phase]).toBe("active"); |
| 4782 | expect(images(predecessor)).toEqual([ |
| 4783 | { name: "sidecar", image: firstImage }, |
| 4784 | { name: "web", image: firstImage }, |
| 4785 | ]); |
| 4786 | yield* stack.destroy(); |
| 4787 | expect(yield* appGone(first.appName)).toBe(true); |
| 4788 | }), |
| 4789 | { timeout: 180_000 }, |
| 4790 | ); |
| 4791 | |
| 4792 | test.provider( |
| 4793 | "tampered deployment metadata preserves the serving group", |
| 4794 | (stack) => |
| 4795 | Effect.gen(function* () { |
| 4796 | yield* stack.destroy(); |
| 4797 | const first = yield* deployGroup(stack, firstImage); |
| 4798 | const target = { app_name: first.appName, machine_id: first.machineId }; |
| 4799 | const original = yield* machines.getMachine(target); |
| 4800 | const tamperings: Array<[string, Record<string, string>]> = [ |
| 4801 | ["unknown protocol", { [keys.protocol]: "future" }], |
| 4802 | [ |
| 4803 | "incomplete image set", |
| 4804 | { |
| 4805 | [keys.containerImageSet]: JSON.stringify([{ name: "web", image: firstImage }]), |
| 4806 | }, |
| 4807 | ], |
| 4808 | ]; |
| 4809 | for (const [label, overrides] of tamperings) { |
| 4810 | // Rewrite Alchemy's ownership metadata directly through the Fly API. |
| 4811 | const current = yield* machines.getMachine(target); |
| 4812 | const tampered = yield* machines.updateMachine({ |
| 4813 | ...target, |
| 4814 | config: { |
| 4815 | ...current.config, |
| 4816 | metadata: { ...original.config?.metadata, ...overrides }, |
| 4817 | }, |
| 4818 | }); |
| 4819 | yield* machines.waitMachine({ |
| 4820 | ...target, |
| 4821 | state: "started", |
| 4822 | instance_id: tampered.instance_id, |
| 4823 | timeout: 30, |
| 4824 | }); |
| 4825 | const failed = yield* deployGroup(stack, nextImage).pipe(Effect.flip); |
| 4826 | expect(failed, label).toBeInstanceOf(DeploymentRecoveryAmbiguous); |
| 4827 | const preserved = yield* machines.getMachine(target); |
| 4828 | expect(preserved.instance_id, label).toBe(tampered.instance_id); |
| 4829 | expect(preserved.state, label).toBe("started"); |
| 4830 | expect(images(preserved), label).toEqual([ |
| 4831 | { name: "sidecar", image: firstImage }, |
| 4832 | { name: "web", image: firstImage }, |
| 4833 | ]); |
| 4834 | const inventory = (yield* machines.listMachines({ |
| 4835 | app_name: first.appName, |
| 4836 | })).filter((machine) => machine.state !== "destroyed"); |
| 4837 | expect( |
| 4838 | inventory.map(({ id }) => id), |
| 4839 | label, |
| 4840 | ).toEqual([first.machineId]); |
| 4841 | } |
| 4842 | const restored = yield* machines.updateMachine({ |
| 4843 | ...target, |
| 4844 | config: { |
| 4845 | ...(yield* machines.getMachine(target)).config, |
| 4846 | metadata: original.config?.metadata, |
| 4847 | }, |
| 4848 | }); |
| 4849 | yield* machines.waitMachine({ |
| 4850 | ...target, |
| 4851 | state: "started", |
| 4852 | instance_id: restored.instance_id, |
| 4853 | timeout: 30, |
| 4854 | }); |
| 4855 | yield* stack.destroy(); |
| 4856 | expect(yield* appGone(first.appName)).toBe(true); |
| 4857 | }), |
| 4858 | { timeout: 180_000 }, |
| 4859 | ); |
| 4860 | }, |
| 4861 | ); |
| 4862 | |
| 4863 | describe.sequential( |
| 4864 | "fiber interruption", |
| 4865 | { |
| 4866 | tags: [ |
| 4867 | "provider:fly", |
| 4868 | "provider:fly:app", |
| 4869 | "provider:fly:machine", |
| 4870 | "provider:fly:service", |
| 4871 | "live", |
| 4872 | ], |
| 4873 | }, |
| 4874 | () => { |
| 4875 | const file = "test/Fly/BlueGreen.test.ts"; |
| 4876 | const { test } = Test.make({ providers: Fly.providers() }); |
| 4877 | |
| 4878 | describe.sequential("in-process engine interruption", () => { |
| 4879 | for (const phase of ["create", "promotion", "retirement"] as const) { |
| 4880 | test.provider( |
| 4881 | `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`, |
| 4882 | (stack) => |
| 4883 | Effect.gen(function* () { |
| 4884 | yield* stack.destroy(); |
| 4885 | const initial = yield* deployWorker(stack, "one"); |
| 4886 | const proxy = yield* transportProxy(); |
| 4887 | const match = (event: TransportEvent) => |
| 4888 | phase === "create" |
| 4889 | ? event.method === "POST" && event.path.endsWith("/machines") |
| 4890 | : phase === "promotion" |
| 4891 | ? event.path.endsWith("/uncordon") |
| 4892 | : event.method === "DELETE" && |
| 4893 | /\/machines\/[^/]+$/.test(event.path) && |
| 4894 | event.machineId === initial.machineId; |
| 4895 | yield* Effect.sync(() => |
| 4896 | proxy.arm({ match, action: "hold-response", remaining: 1 }), |
| 4897 | ); |
| 4898 | yield* Effect.gen(function* () { |
| 4899 | const actor = yield* engineActor( |
| 4900 | stack, |
| 4901 | `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`, |
| 4902 | file, |
| 4903 | proxy.url, |
| 4904 | ); |
| 4905 | const interrupted = yield* deployWorker(actor, "two").pipe( |
| 4906 | Effect.scoped, |
| 4907 | Effect.forkScoped, |
| 4908 | ); |
| 4909 | const barrier = yield* proxy.wait( |
| 4910 | (event) => |
| 4911 | event.stage === "held" && |
| 4912 | event.status! >= 200 && |
| 4913 | event.status! < 300 && |
| 4914 | match(event), |
| 4915 | ); |
| 4916 | expect(barrier.machineId).toBeDefined(); |
| 4917 | const interruption = yield* Fiber.interrupt(interrupted).pipe( |
| 4918 | Effect.forkScoped, |
| 4919 | ); |
| 4920 | yield* Effect.yieldNow; |
| 4921 | // Let accepted uninterruptible work and finalizers settle; this is not process-kill evidence. |
| 4922 | yield* Effect.sync(() => { |
| 4923 | proxy.clear(); |
| 4924 | proxy.release(); |
| 4925 | }); |
| 4926 | yield* Fiber.join(interruption).pipe(Effect.timeout("90 seconds")); |
| 4927 | const exit = yield* Fiber.await(interrupted); |
| 4928 | expect(Exit.hasInterrupts(exit)).toBe(true); |
| 4929 | const surviving = yield* census(initial.appName); |
| 4930 | const green = surviving.filter((machine) => machine.id !== initial.machineId); |
| 4931 | if (phase === "create") { |
| 4932 | expect(surviving.some((machine) => machine.id === initial.machineId)).toBe( |
| 4933 | true, |
| 4934 | ); |
| 4935 | expect(green.length).toBeLessThanOrEqual(1); |
| 4936 | } else { |
| 4937 | expect(green).toHaveLength(1); |
| 4938 | expect(green[0]!.cordoned).toBe(false); |
| 4939 | } |
| 4940 | const resumed = yield* engineActor( |
| 4941 | stack, |
| 4942 | `${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`, |
| 4943 | file, |
| 4944 | ); |
| 4945 | expect(resumed.state).not.toBe(actor.state); |
| 4946 | const recovered = yield* deployWorker(resumed, "two").pipe(Effect.scoped); |
| 4947 | if (green.length) |
| 4948 | expect(recovered.machineIds).toEqual(green.map((machine) => machine.id)); |
| 4949 | yield* assertCommitted(initial.appName, recovered.machineIds); |
| 4950 | }).pipe( |
| 4951 | Effect.ensuring( |
| 4952 | Effect.sync(() => { |
| 4953 | proxy.clear(); |
| 4954 | proxy.release(); |
| 4955 | }), |
| 4956 | ), |
| 4957 | Effect.scoped, |
| 4958 | ); |
| 4959 | yield* stack.destroy(); |
| 4960 | yield* assertAppGone(initial.appName); |
| 4961 | }).pipe(Effect.scoped), |
| 4962 | { timeout: 300_000 }, |
| 4963 | ); |
| 4964 | } |
| 4965 | }); |
| 4966 | }, |
| 4967 | ); |
| 4968 | |
| 4969 | describe.sequential( |
| 4970 | "multi-container public traffic", |
| 4971 | { tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"] }, |
| 4972 | () => { |
| 4973 | const { test } = Test.make({ providers: Fly.providers() }); |
| 4974 | const image = |
| 4975 | "docker-hub-mirror.fly.io/library/node@sha256:b6f26b36c8ff49624cfdac716b8ea1138d606df02586a77d364bb5536a634f85"; |
| 4976 | |
| 4977 | const scenario = (stack: ScratchStack) => |
| 4978 | Effect.gen(function* () { |
| 4979 | const fs = yield* FileSystem.FileSystem; |
| 4980 | const path = yield* Path.Path; |
| 4981 | const script = yield* fs.readFileString( |
| 4982 | yield* path.fromFileUrl( |
| 4983 | new URL("./fixtures/multi-container-http.ts", import.meta.url), |
| 4984 | ), |
| 4985 | ); |
| 4986 | return (version: string, badHealth = false) => |
| 4987 | stack.deploy( |
| 4988 | Effect.gen(function* () { |
| 4989 | const app = yield* Fly.App("Traffic"); |
| 4990 | yield* Fly.IpAssignment("Public", { app, type: "shared_v4" }); |
| 4991 | return yield* Fly.Machine("Group", { |
| 4992 | app, |
| 4993 | region: "fra", |
| 4994 | guest: { cpus: 1, memoryMb: 256 }, |
| 4995 | containers: [ |
| 4996 | { |
| 4997 | name: "web", |
| 4998 | image, |
| 4999 | cmd: ["node", "--input-type=module-typescript", "-e", script], |
| 5000 | env: { PORT: "3000", CONTAINER_NAME: "web", VERSION: version }, |
Showing first 5,000 of 9,884 lines.
View raw