The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as Fly from "@alchemy.run/fly"; |
| 2 | import { makeMachineLeases } from "@alchemy.run/fly/leases"; |
| 3 | import { describe, expect } from "@alchemy.run/test"; |
| 4 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 5 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 6 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 7 | import * as Clock from "effect/Clock"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Fiber from "effect/Fiber"; |
| 10 | import * as HttpClient from "effect/http/HttpClient"; |
| 11 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 12 | import * as Result from "effect/Result"; |
| 13 | import { assertAppGone } from "./fixtures/bluegreen.ts"; |
| 14 | import { transportProxy, type TransportEvent } from "./fixtures/transport.ts"; |
| 15 | |
| 16 | const { test } = Test.make({ providers: Fly.providers() }); |
| 17 | |
| 18 | const targets = Effect.gen(function* () { |
| 19 | const app = yield* Fly.App("LeaseSite"); |
| 20 | return yield* Fly.Machine("LeaseTargets", { |
| 21 | app, |
| 22 | region: "iad", |
| 23 | image: "nginx:alpine", |
| 24 | guest: { cpus: 1, memoryMb: 256 }, |
| 25 | count: 2, |
| 26 | skipLaunch: true, |
| 27 | }); |
| 28 | }); |
| 29 | |
| 30 | const sanitizeFailure = <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| 31 | effect.pipe( |
| 32 | Effect.mapError( |
| 33 | (error) => new Error(error instanceof Error ? error.name : "Fly lease regression failed"), |
| 34 | ), |
| 35 | ); |
| 36 | |
| 37 | const isLease = (machineId: string) => (event: TransportEvent) => |
| 38 | event.method === "POST" && event.path.endsWith(`/machines/${machineId}/lease`); |
| 39 | |
| 40 | const isAbsent = (event: TransportEvent) => event.status === 404 || event.state === "destroyed"; |
| 41 | |
| 42 | // Faults hold or drop real responses; no Fly status, lease, or nonce is fabricated. |
| 43 | describe.sequential( |
| 44 | "native lease controller concurrency", |
| 45 | { tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"] }, |
| 46 | () => { |
| 47 | for (const confirmed of [true, false]) { |
| 48 | test.provider( |
| 49 | confirmed |
| 50 | ? "FLY-REVIEW-2 a late absence timeout cannot override confirmed removal" |
| 51 | : "FLY-REVIEW-2 unconfirmed removal still cancels shared authority", |
| 52 | (stack) => |
| 53 | Effect.gen(function* () { |
| 54 | yield* stack.destroy(); |
| 55 | const created = yield* stack.deploy(targets); |
| 56 | const [removedId, siblingId] = [...created.machineIds].sort(); |
| 57 | expect(created.machineIds).toHaveLength(2); |
| 58 | const target = { app_name: created.appName, machine_id: removedId! }; |
| 59 | const sibling = { app_name: created.appName, machine_id: siblingId! }; |
| 60 | yield* Effect.gen(function* () { |
| 61 | const first = yield* transportProxy(); |
| 62 | const second = yield* transportProxy(); |
| 63 | const client = yield* HttpClient.HttpClient; |
| 64 | let observations = 0; |
| 65 | const routed = HttpClient.mapRequest(client, (request) => { |
| 66 | if (!request.url.startsWith("https://api.machines.dev/")) return request; |
| 67 | const observing = |
| 68 | request.method === "GET" && request.url.endsWith(`/machines/${removedId}`); |
| 69 | const proxy = observing && ++observations > 1 ? second : first; |
| 70 | return HttpClientRequest.setUrl( |
| 71 | request, |
| 72 | request.url.replace("https://api.machines.dev", proxy.url), |
| 73 | ); |
| 74 | }); |
| 75 | yield* Effect.gen(function* () { |
| 76 | const leases = yield* makeMachineLeases(created.appName); |
| 77 | yield* leases.acquire(created.machineIds); |
| 78 | const before = yield* machines.getMachineLease(sibling).pipe(Retry.none); |
| 79 | const deleting = (event: TransportEvent) => |
| 80 | event.method === "DELETE" && event.path.endsWith(`/machines/${removedId}`); |
| 81 | const observing = (event: TransportEvent) => |
| 82 | event.method === "GET" && event.path.endsWith(`/machines/${removedId}`); |
| 83 | yield* Effect.sync(() => { |
| 84 | first.arm({ match: deleting, action: "drop-response", remaining: 1 }); |
| 85 | first.arm({ match: observing, action: "hold-response", remaining: 1 }); |
| 86 | second.arm({ match: observing, action: "hold-response", remaining: Infinity }); |
| 87 | }); |
| 88 | // Start absence observation just before the target's first renewal. |
| 89 | yield* leases.guard(Effect.sleep("20 seconds")); |
| 90 | const removal = yield* leases |
| 91 | .remove(removedId!, (lease_nonce) => |
| 92 | machines.deleteMachine({ ...target, lease_nonce, force: true }), |
| 93 | ) |
| 94 | .pipe(Effect.result, Effect.forkScoped); |
| 95 | const dropped = yield* first.wait( |
| 96 | (event) => deleting(event) && event.stage === "dropped", |
| 97 | ); |
| 98 | expect(dropped.status! >= 200 && dropped.status! < 300).toBe(true); |
| 99 | const initialObservation = yield* first.wait( |
| 100 | (event) => observing(event) && event.stage === "held", |
| 101 | ); |
| 102 | expect(isAbsent(initialObservation)).toBe(true); |
| 103 | const lateObservation = yield* second.wait( |
| 104 | (event) => observing(event) && event.stage === "held", |
| 105 | ); |
| 106 | expect(isAbsent(lateObservation)).toBe(true); |
| 107 | expect( |
| 108 | first.events.some( |
| 109 | (event) => |
| 110 | isLease(removedId!)(event) && |
| 111 | event.stage === "forwarded" && |
| 112 | event.status === 404, |
| 113 | ), |
| 114 | ).toBe(true); |
| 115 | if (confirmed) { |
| 116 | yield* Effect.sync(first.release); |
| 117 | const removed = yield* Fiber.join(removal).pipe(Effect.timeout("5 seconds")); |
| 118 | expect(Result.isSuccess(removed)).toBe(true); |
| 119 | expect(yield* leases.nonceIfHeld(removedId!)).toBeUndefined(); |
| 120 | } |
| 121 | // The renewal observer remains held until its own 30-second deadline. |
| 122 | const outcome = yield* leases.guard(Effect.sleep("32 seconds")).pipe(Effect.result); |
| 123 | expect(Result.isSuccess(outcome)).toBe(confirmed); |
| 124 | if (Result.isFailure(outcome)) { |
| 125 | expect(outcome.failure._tag).toBe("Fly.MachineLeaseLost"); |
| 126 | expect(outcome.failure.machineId).toBe(removedId); |
| 127 | expect(outcome.failure.reason).toBe( |
| 128 | "lease disappeared before removal was confirmed", |
| 129 | ); |
| 130 | } |
| 131 | expect( |
| 132 | second.events.some((event) => observing(event) && event.stage === "forwarded"), |
| 133 | ).toBe(false); |
| 134 | if (confirmed) { |
| 135 | yield* leases.checkTarget(siblingId!); |
| 136 | const after = yield* machines.getMachineLease(sibling).pipe(Retry.none); |
| 137 | expect(after.data?.expires_at).toBeGreaterThan(before.data!.expires_at!); |
| 138 | expect( |
| 139 | first.events.some( |
| 140 | (event) => |
| 141 | isLease(siblingId!)(event) && |
| 142 | event.stage === "forwarded" && |
| 143 | event.sequence > dropped.sequence && |
| 144 | event.status! >= 200 && |
| 145 | event.status! < 300, |
| 146 | ), |
| 147 | ).toBe(true); |
| 148 | } else { |
| 149 | const removed = yield* Fiber.join(removal).pipe(Effect.timeout("5 seconds")); |
| 150 | expect(Result.isFailure(removed)).toBe(true); |
| 151 | } |
| 152 | expect( |
| 153 | first.events.filter((event) => deleting(event) && event.stage === "request"), |
| 154 | ).toHaveLength(1); |
| 155 | }).pipe(Effect.scoped, Effect.provideService(HttpClient.HttpClient, routed)); |
| 156 | }).pipe(Effect.scoped); |
| 157 | yield* stack.destroy(); |
| 158 | yield* assertAppGone(created.appName); |
| 159 | }).pipe(sanitizeFailure), |
| 160 | { timeout: 300_000 }, |
| 161 | ); |
| 162 | } |
| 163 | |
| 164 | test.provider( |
| 165 | "FLY-REVIEW-4 held renewal responses do not consume another target's renewal interval", |
| 166 | (stack) => |
| 167 | Effect.gen(function* () { |
| 168 | yield* stack.destroy(); |
| 169 | const created = yield* stack.deploy(targets); |
| 170 | const [a, b] = [...created.machineIds].sort(); |
| 171 | expect(created.machineIds).toHaveLength(2); |
| 172 | yield* Effect.gen(function* () { |
| 173 | const first = yield* transportProxy(); |
| 174 | const second = yield* transportProxy(); |
| 175 | const client = yield* HttpClient.HttpClient; |
| 176 | const routed = HttpClient.mapRequest(client, (request) => { |
| 177 | if (!request.url.startsWith("https://api.machines.dev/")) return request; |
| 178 | const proxy = request.url.includes(`/machines/${a}/`) ? first : second; |
| 179 | return HttpClientRequest.setUrl( |
| 180 | request, |
| 181 | request.url.replace("https://api.machines.dev", proxy.url), |
| 182 | ); |
| 183 | }); |
| 184 | yield* Effect.gen(function* () { |
| 185 | const leases = yield* makeMachineLeases(created.appName); |
| 186 | yield* leases.acquire(created.machineIds); |
| 187 | let renewalsA = 0; |
| 188 | yield* Effect.sync(() => { |
| 189 | first.arm({ |
| 190 | match: (event) => isLease(a!)(event) && ++renewalsA === 2, |
| 191 | action: "hold-response", |
| 192 | remaining: 1, |
| 193 | }); |
| 194 | second.arm({ match: isLease(b!), action: "hold-response", remaining: 1 }); |
| 195 | }); |
| 196 | const heldB = yield* leases.guard( |
| 197 | second.wait((event) => isLease(b!)(event) && event.stage === "held"), |
| 198 | ); |
| 199 | expect(heldB.status! >= 200 && heldB.status! < 300).toBe(true); |
| 200 | const heldAt = yield* Clock.currentTimeMillis; |
| 201 | const releaseB = yield* Effect.sleep("60 seconds").pipe( |
| 202 | Effect.andThen(Effect.sync(second.release)), |
| 203 | Effect.forkScoped, |
| 204 | ); |
| 205 | const heldA = yield* leases |
| 206 | .guard(first.wait((event) => isLease(a!)(event) && event.stage === "held")) |
| 207 | .pipe(Effect.timeout("45 seconds")); |
| 208 | expect(heldA.status! >= 200 && heldA.status! < 300).toBe(true); |
| 209 | expect((yield* Clock.currentTimeMillis) - heldAt).toBeLessThan(45_000); |
| 210 | expect( |
| 211 | second.events.some( |
| 212 | (event) => event.sequence === heldB.sequence && event.stage === "forwarded", |
| 213 | ), |
| 214 | ).toBe(false); |
| 215 | const releaseA = yield* Effect.sleep("60 seconds").pipe( |
| 216 | Effect.andThen(Effect.sync(first.release)), |
| 217 | Effect.forkScoped, |
| 218 | ); |
| 219 | yield* leases.guard(Effect.all([Fiber.join(releaseA), Fiber.join(releaseB)])); |
| 220 | yield* leases.guard(Effect.sleep("5 seconds")); |
| 221 | for (const machineId of created.machineIds) yield* leases.checkTarget(machineId); |
| 222 | for (const [proxy, held, machineId] of [ |
| 223 | [first, heldA, a!], |
| 224 | [second, heldB, b!], |
| 225 | ] as const) { |
| 226 | const forwarded = proxy.events.findIndex( |
| 227 | (event) => event.sequence === held.sequence && event.stage === "forwarded", |
| 228 | ); |
| 229 | const started = proxy.events.findIndex( |
| 230 | (event) => event.sequence === held.sequence && event.stage === "held", |
| 231 | ); |
| 232 | expect(forwarded).toBeGreaterThan(started); |
| 233 | expect( |
| 234 | proxy.events |
| 235 | .slice(started + 1, forwarded) |
| 236 | .some((event) => isLease(machineId)(event) && event.stage === "request"), |
| 237 | ).toBe(false); |
| 238 | expect( |
| 239 | proxy.events |
| 240 | .slice(forwarded + 1) |
| 241 | .some( |
| 242 | (event) => |
| 243 | isLease(machineId)(event) && |
| 244 | event.stage === "forwarded" && |
| 245 | event.status! >= 200 && |
| 246 | event.status! < 300, |
| 247 | ), |
| 248 | ).toBe(true); |
| 249 | } |
| 250 | }).pipe(Effect.scoped, Effect.provideService(HttpClient.HttpClient, routed)); |
| 251 | }).pipe(Effect.scoped); |
| 252 | yield* stack.destroy(); |
| 253 | yield* assertAppGone(created.appName); |
| 254 | }).pipe(sanitizeFailure), |
| 255 | { timeout: 300_000 }, |
| 256 | ); |
| 257 | }, |
| 258 | ); |