The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import { expect } from "@alchemy.run/test"; |
| 2 | import type { ScratchStack } from "@alchemy.run/test/Alchemy"; |
| 3 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 4 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as FileSystem from "effect/FileSystem"; |
| 7 | import * as Path from "effect/Path"; |
| 8 | import * as Schedule from "effect/Schedule"; |
| 9 | import * as Schema from "effect/Schema"; |
| 10 | import { |
| 11 | Witness as ProcessDeathWitness, |
| 12 | census, |
| 13 | evidencePaths, |
| 14 | identity, |
| 15 | machine, |
| 16 | nowSeconds, |
| 17 | persistedRow, |
| 18 | } from "./process-death.ts"; |
| 19 | import type { TransportEvent } from "./transport.ts"; |
| 20 | |
| 21 | export const signalOverlapFile = "test/Fly/BlueGreen.test.ts"; |
| 22 | export const cases = [ |
| 23 | { name: "sigint-create", signal: "SIGINT", phase: "create" }, |
| 24 | { name: "sigint-promotion", signal: "SIGINT", phase: "promotion" }, |
| 25 | { name: "sigint-overlap", signal: "SIGINT", phase: "overlap" }, |
| 26 | { name: "sigint-retirement", signal: "SIGINT", phase: "retirement" }, |
| 27 | { name: "sigkill-overlap", signal: "SIGKILL", phase: "overlap" }, |
| 28 | ] as const; |
| 29 | export type SignalCase = (typeof cases)[number]; |
| 30 | |
| 31 | const Boundary = Schema.Struct({ |
| 32 | sequence: Schema.Number, |
| 33 | method: Schema.String, |
| 34 | path: Schema.String, |
| 35 | status: Schema.Number, |
| 36 | machineId: Schema.String, |
| 37 | }); |
| 38 | |
| 39 | export const Witness = Schema.Struct({ |
| 40 | ...ProcessDeathWitness.fields, |
| 41 | signal: Schema.Literals(["SIGINT", "SIGKILL"]), |
| 42 | phase: Schema.Literals(["create", "promotion", "overlap", "retirement"]), |
| 43 | case: Schema.String, |
| 44 | barrier: Boundary, |
| 45 | // The subsequent routing read proves the provider consumed the uncordon response. |
| 46 | returnedUncordon: Schema.optional(Boundary), |
| 47 | }); |
| 48 | export type Witness = typeof Witness.Type; |
| 49 | |
| 50 | const LeaseObservation = Schema.Struct({ |
| 51 | machineId: Schema.String, |
| 52 | present: Schema.Boolean, |
| 53 | expiresAt: Schema.Number, |
| 54 | at: Schema.Number, |
| 55 | }); |
| 56 | |
| 57 | export const Finalized = Schema.Struct({ |
| 58 | pid: Schema.Number, |
| 59 | case: Schema.String, |
| 60 | witnessRecordedAt: Schema.Number, |
| 61 | interruptedOnly: Schema.Boolean, |
| 62 | at: Schema.Number, |
| 63 | releases: Schema.Array(Boundary), |
| 64 | leases: Schema.Array(LeaseObservation), |
| 65 | }); |
| 66 | |
| 67 | export const RunnerInterrupted = Schema.Struct({ |
| 68 | pid: Schema.Number, |
| 69 | case: Schema.String, |
| 70 | signal: Schema.Literal("SIGINT"), |
| 71 | witnessRecordedAt: Schema.Number, |
| 72 | at: Schema.Number, |
| 73 | deployFinalized: Schema.Boolean, |
| 74 | }); |
| 75 | |
| 76 | export const pathsFor = (stack: ScratchStack) => |
| 77 | Effect.gen(function* () { |
| 78 | const paths = yield* evidencePaths(stack); |
| 79 | const path = yield* Path.Path; |
| 80 | return { |
| 81 | ...paths, |
| 82 | runnerInterrupted: path.join(paths.directory, "runner-interrupted.json"), |
| 83 | blocked: path.join(paths.directory, "interruption-limit.json"), |
| 84 | }; |
| 85 | }); |
| 86 | |
| 87 | export const readEvidence = <S extends Schema.Top>(file: string, schema: S) => |
| 88 | Effect.gen(function* () { |
| 89 | const fs = yield* FileSystem.FileSystem; |
| 90 | return yield* fs.readFileString(file).pipe( |
| 91 | Effect.flatMap( |
| 92 | Schema.decodeUnknownEffect(Schema.fromJsonString(schema), { |
| 93 | onExcessProperty: "error", |
| 94 | }), |
| 95 | ), |
| 96 | Effect.mapError(() => new Error("Missing or invalid signal evidence; preserve the attempt")), |
| 97 | ); |
| 98 | }); |
| 99 | |
| 100 | export const assertSingleRunner = Effect.gen(function* () { |
| 101 | const args = yield* Effect.sync(() => process.argv.slice(2)); |
| 102 | const focused = args[0] === "--exclude" && args[1] === "test/Railway" ? args.slice(2) : args; |
| 103 | expect(focused).toEqual([ |
| 104 | signalOverlapFile, |
| 105 | "-t", |
| 106 | " > process signals and overlap > ", |
| 107 | "--profile", |
| 108 | "testing", |
| 109 | "--retry", |
| 110 | "0", |
| 111 | "--concurrency", |
| 112 | "1", |
| 113 | "--sequential", |
| 114 | "--timeout", |
| 115 | "750000", |
| 116 | ]); |
| 117 | expect(process.env.FLY_SIGNAL_OVERLAP_ONLY_RUNNER).toBe("1"); |
| 118 | expect(process.env.FLY_PROCESS_DEATH_CASE).toBeUndefined(); |
| 119 | expect(process.env.FLY_PROCESS_DEATH_MODE).toBeUndefined(); |
| 120 | expect(process.env.ALCHEMY_PROFILE).toBe("testing"); |
| 121 | expect(process.env.ALCHEMY_TEST_DEV).toBe("0"); |
| 122 | expect(process.env.ALCHEMY_DEV).toBe("0"); |
| 123 | expect(process.env.ALCHEMY_TEST_STAGE).toBe("test_fly_signal_overlap"); |
| 124 | expect(process.env.FAST).toBeUndefined(); |
| 125 | expect(process.versions.bun).toBeDefined(); |
| 126 | const fs = yield* FileSystem.FileSystem; |
| 127 | for (const file of [ |
| 128 | signalOverlapFile, |
| 129 | "test/Fly/fixtures/signal-overlap.ts", |
| 130 | "test/Fly/fixtures/process-death.ts", |
| 131 | "test/Fly/fixtures/transport.ts", |
| 132 | ]) { |
| 133 | const source = yield* fs.readFileString(file); |
| 134 | expect(source).not.toMatch( |
| 135 | /(?:from\s+|import\s*(?:\(\s*)?|require\s*\()\s*["'](?:node:)?child_process["']/, |
| 136 | ); |
| 137 | expect(source).not.toMatch(/\b(?:Bun|Deno)\s*\.\s*(?:spawn|spawnSync|Command)\s*\(/); |
| 138 | expect(source).not.toMatch(/from\s+["'][^"']*(?:ChildProcess|CommandExecutor)[^"']*["']/); |
| 139 | } |
| 140 | }); |
| 141 | |
| 142 | export const successful = (event: TransportEvent) => |
| 143 | event.status !== undefined && event.status >= 200 && event.status < 300; |
| 144 | |
| 145 | export const boundary = (event: TransportEvent) => { |
| 146 | expect(successful(event)).toBe(true); |
| 147 | expect(event.machineId).toBeDefined(); |
| 148 | return { |
| 149 | sequence: event.sequence, |
| 150 | method: event.method, |
| 151 | path: event.path, |
| 152 | status: event.status!, |
| 153 | machineId: event.machineId!, |
| 154 | }; |
| 155 | }; |
| 156 | |
| 157 | export const firstReturnedUncordon = (events: readonly TransportEvent[], appName: string) => |
| 158 | events.find( |
| 159 | (event) => |
| 160 | event.stage === "forwarded" && |
| 161 | successful(event) && |
| 162 | event.method === "POST" && |
| 163 | event.path.startsWith(`/v1/apps/${appName}/machines/`) && |
| 164 | event.path.endsWith("/uncordon"), |
| 165 | ); |
| 166 | |
| 167 | export const matches = ( |
| 168 | phase: SignalCase["phase"], |
| 169 | appName: string, |
| 170 | predecessorId: string, |
| 171 | events: readonly TransportEvent[], |
| 172 | event: TransportEvent, |
| 173 | ) => { |
| 174 | const base = `/v1/apps/${appName}/machines`; |
| 175 | if (phase === "create") return event.method === "POST" && event.path === base; |
| 176 | if (phase === "promotion") |
| 177 | return ( |
| 178 | event.method === "POST" && |
| 179 | event.path.startsWith(`${base}/`) && |
| 180 | event.path.endsWith("/uncordon") |
| 181 | ); |
| 182 | if (phase === "retirement") |
| 183 | return event.method === "DELETE" && event.path === `${base}/${predecessorId}`; |
| 184 | const returned = firstReturnedUncordon(events, appName); |
| 185 | return ( |
| 186 | returned !== undefined && |
| 187 | returned.machineId !== predecessorId && |
| 188 | event.sequence > returned.sequence && |
| 189 | event.method === "GET" && |
| 190 | event.path === `${base}/${returned.machineId}` |
| 191 | ); |
| 192 | }; |
| 193 | |
| 194 | export const assertBoundary = (witness: Witness) => |
| 195 | Effect.sync(() => { |
| 196 | const { barrier, returnedUncordon } = witness; |
| 197 | expect(barrier.sequence).toBeGreaterThan(0); |
| 198 | expect(barrier.status).toBeGreaterThanOrEqual(200); |
| 199 | expect(barrier.status).toBeLessThan(300); |
| 200 | expect(barrier.machineId).toBe( |
| 201 | witness.phase === "retirement" ? witness.predecessor.id : witness.candidate.id, |
| 202 | ); |
| 203 | if (witness.phase === "overlap") { |
| 204 | expect(returnedUncordon).toBeDefined(); |
| 205 | expect(returnedUncordon!.method).toBe("POST"); |
| 206 | expect(returnedUncordon!.path).toBe( |
| 207 | `/v1/apps/${witness.appName}/machines/${witness.candidate.id}/uncordon`, |
| 208 | ); |
| 209 | expect(returnedUncordon!.machineId).toBe(witness.candidate.id); |
| 210 | expect(returnedUncordon!.status).toBeGreaterThanOrEqual(200); |
| 211 | expect(returnedUncordon!.status).toBeLessThan(300); |
| 212 | expect(returnedUncordon!.sequence).toBeGreaterThan(0); |
| 213 | expect(barrier.sequence).toBeGreaterThan(returnedUncordon!.sequence); |
| 214 | } else expect(returnedUncordon).toBeUndefined(); |
| 215 | expect( |
| 216 | matches( |
| 217 | witness.phase, |
| 218 | witness.appName, |
| 219 | witness.predecessor.id, |
| 220 | returnedUncordon ? [{ ...returnedUncordon, stage: "forwarded" }] : [], |
| 221 | { ...barrier, stage: "held" }, |
| 222 | ), |
| 223 | ).toBe(true); |
| 224 | }); |
| 225 | |
| 226 | export const assertInventory = (stack: ScratchStack, witness: Witness) => |
| 227 | Effect.gen(function* () { |
| 228 | const live = yield* census(witness.appName); |
| 229 | expect(live.map((value) => value.id).sort()).toEqual( |
| 230 | (witness.phase === "retirement" |
| 231 | ? [witness.candidate.id] |
| 232 | : [witness.predecessor.id, witness.candidate.id] |
| 233 | ).sort(), |
| 234 | ); |
| 235 | const candidate = yield* machine(witness.appName, witness.candidate.id); |
| 236 | expect(yield* identity(candidate, stack)).toEqual(witness.candidate); |
| 237 | expect(candidate.config?.env?.VERSION).toBe("two"); |
| 238 | expect(candidate.cordoned).toBe(witness.phase === "create"); |
| 239 | expect(witness.candidate.phase).toBe( |
| 240 | witness.phase === "create" |
| 241 | ? "candidate" |
| 242 | : witness.phase === "retirement" |
| 243 | ? "active" |
| 244 | : "promoting", |
| 245 | ); |
| 246 | if (witness.phase !== "retirement") { |
| 247 | const predecessor = yield* machine(witness.appName, witness.predecessor.id); |
| 248 | expect(yield* identity(predecessor, stack)).toEqual(witness.predecessor); |
| 249 | expect(predecessor.cordoned).toBe(false); |
| 250 | expect(predecessor.state).toBe("started"); |
| 251 | expect(predecessor.config?.env?.VERSION).toBe("one"); |
| 252 | } |
| 253 | }); |
| 254 | |
| 255 | // Only presence and timestamps escape the SDK; nonce and transport headers never do. |
| 256 | export const observeLease = (appName: string, machineId: string) => |
| 257 | machines.getMachineLease({ app_name: appName, machine_id: machineId }).pipe( |
| 258 | Retry.none, |
| 259 | Effect.map((value) => ({ |
| 260 | present: !!value.data?.nonce, |
| 261 | expiresAt: value.data?.expires_at ?? 0, |
| 262 | })), |
| 263 | Effect.catchTag("NotFound", () => Effect.succeed({ present: false, expiresAt: 0 })), |
| 264 | Effect.timeout("10 seconds"), |
| 265 | Effect.mapError((error) => new Error(`Signal lease observation failed (${error._tag})`)), |
| 266 | Effect.flatMap((value) => nowSeconds.pipe(Effect.map((at) => ({ machineId, ...value, at })))), |
| 267 | ); |
| 268 | |
| 269 | export const observeLeases = (witness: Witness) => |
| 270 | Effect.forEach(witness.leases, (held) => observeLease(witness.appName, held.machineId), { |
| 271 | concurrency: 2, |
| 272 | }); |
| 273 | |
| 274 | export const observeExpiry = (witness: Witness) => |
| 275 | Effect.gen(function* () { |
| 276 | expect(witness.signal).toBe("SIGKILL"); |
| 277 | expect(witness.leases.length).toBeGreaterThan(0); |
| 278 | const initial = yield* observeLeases(witness); |
| 279 | for (const [index, value] of initial.entries()) { |
| 280 | expect(value.present).toBe(true); |
| 281 | expect(value.expiresAt).toBeGreaterThan(value.at); |
| 282 | expect(value.expiresAt).toBeGreaterThanOrEqual(witness.leases[index]!.expiresAt); |
| 283 | expect(value.expiresAt).toBeLessThanOrEqual(witness.recordedAt + 130); |
| 284 | } |
| 285 | const samples: (typeof LeaseObservation.Type)[] = []; |
| 286 | const expired = yield* Effect.forEach( |
| 287 | initial, |
| 288 | (held) => |
| 289 | Effect.gen(function* () { |
| 290 | const value = yield* observeLease(witness.appName, held.machineId); |
| 291 | yield* Effect.sync(() => samples.push(value)); |
| 292 | if (value.present) expect(value.expiresAt).toBe(held.expiresAt); |
| 293 | else expect(value.at).toBeGreaterThanOrEqual(held.expiresAt); |
| 294 | return !value.present; |
| 295 | }), |
| 296 | { concurrency: 2 }, |
| 297 | ).pipe( |
| 298 | Effect.repeat({ |
| 299 | schedule: Schedule.spaced("3 seconds"), |
| 300 | times: 65, |
| 301 | until: (all) => all.every(Boolean), |
| 302 | }), |
| 303 | Effect.timeout("230 seconds"), |
| 304 | ); |
| 305 | expect(expired.every(Boolean)).toBe(true); |
| 306 | for (const held of initial) { |
| 307 | const surviving = yield* machine(witness.appName, held.machineId); |
| 308 | expect(surviving.state).not.toBe("destroyed"); |
| 309 | } |
| 310 | return { initial, samples }; |
| 311 | }); |
| 312 | |
| 313 | export const assertReleased = (witness: Witness, finalized: typeof Finalized.Type) => |
| 314 | Effect.gen(function* () { |
| 315 | expect(witness.signal).toBe("SIGINT"); |
| 316 | expect(witness.leases.length).toBeGreaterThan(0); |
| 317 | expect(finalized.pid).toBe(witness.pid); |
| 318 | expect(finalized.case).toBe(witness.case); |
| 319 | expect(finalized.witnessRecordedAt).toBe(witness.recordedAt); |
| 320 | expect(finalized.interruptedOnly).toBe(true); |
| 321 | expect(finalized.at).toBeGreaterThanOrEqual(witness.recordedAt); |
| 322 | expect(finalized.leases.map((value) => value.machineId).sort()).toEqual( |
| 323 | witness.leases.map((value) => value.machineId).sort(), |
| 324 | ); |
| 325 | for (const held of witness.leases) { |
| 326 | const observed = finalized.leases.find((value) => value.machineId === held.machineId)!; |
| 327 | expect(observed.present).toBe(false); |
| 328 | expect(observed.at).toBeGreaterThanOrEqual(witness.recordedAt); |
| 329 | expect(observed.at).toBeLessThan(held.expiresAt); |
| 330 | expect(finalized.at).toBeLessThan(held.expiresAt); |
| 331 | const release = finalized.releases.find( |
| 332 | (event) => event.machineId === held.machineId && event.status >= 200 && event.status < 300, |
| 333 | ); |
| 334 | expect(release).toBeDefined(); |
| 335 | expect(release!.method).toBe("DELETE"); |
| 336 | expect(release!.path).toBe(`/v1/apps/${witness.appName}/machines/${held.machineId}/lease`); |
| 337 | expect(release!.sequence).toBeGreaterThan(witness.barrier.sequence); |
| 338 | // A deleted Machine cannot masquerade as a released lease. |
| 339 | const surviving = yield* machine(witness.appName, held.machineId); |
| 340 | expect(surviving.state).not.toBe("destroyed"); |
| 341 | } |
| 342 | const current = yield* observeLeases(witness); |
| 343 | expect(current.every((value) => !value.present)).toBe(true); |
| 344 | return { finalized, current }; |
| 345 | }); |
| 346 | |
| 347 | export const assertConverged = (stack: ScratchStack, witness: Witness, ids: string[]) => |
| 348 | Effect.gen(function* () { |
| 349 | expect(ids).toEqual([witness.candidate.id]); |
| 350 | const live = yield* census(witness.appName); |
| 351 | expect(live.map((value) => value.id)).toEqual(ids); |
| 352 | const current = yield* machine(witness.appName, witness.candidate.id).pipe( |
| 353 | Effect.repeat({ |
| 354 | schedule: Schedule.spaced("2 seconds"), |
| 355 | times: 20, |
| 356 | until: (value) => |
| 357 | value.state === "started" && |
| 358 | value.cordoned === false && |
| 359 | value.checks?.some((check) => check.name === "ready" && check.status === "passing") === |
| 360 | true, |
| 361 | }), |
| 362 | Effect.timeout("60 seconds"), |
| 363 | ); |
| 364 | expect(yield* identity(current, stack)).toEqual({ |
| 365 | ...witness.candidate, |
| 366 | phase: "active", |
| 367 | }); |
| 368 | expect(current.state).toBe("started"); |
| 369 | expect(current.cordoned).toBe(false); |
| 370 | expect(current.config?.env?.VERSION).toBe("two"); |
| 371 | const ready = current.checks?.filter((check) => check.name === "ready") ?? []; |
| 372 | expect(ready.length).toBeGreaterThan(0); |
| 373 | expect(ready.every((check) => check.status === "passing")).toBe(true); |
| 374 | expect(current.instance_id).toBeDefined(); |
| 375 | expect(current.config?.metadata?.["alchemy.checked-instance"]).toBe(current.instance_id); |
| 376 | expect(current.config?.metadata?.["alchemy.predecessors"]).toBe(witness.predecessor.id); |
| 377 | expect( |
| 378 | current.config?.metadata?.["alchemy.image"]?.endsWith(`@${witness.candidate.digest}`), |
| 379 | ).toBe(true); |
| 380 | const row = yield* persistedRow(stack, witness.row.fqn); |
| 381 | expect(row.instanceId).toBe(witness.row.instanceId); |
| 382 | expect(["created", "updated"]).toContain(row.status); |
| 383 | }); |