The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as Fly from "@alchemy.run/fly"; |
| 2 | import { expect } from "@alchemy.run/test"; |
| 3 | import type { ScratchStack } from "@alchemy.run/test/Alchemy"; |
| 4 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 5 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 6 | import { State } from "alchemy/State/State"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as FileSystem from "effect/FileSystem"; |
| 9 | import * as Path from "effect/Path"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Schema from "effect/Schema"; |
| 12 | import type { TransportEvent } from "./transport.ts"; |
| 13 | |
| 14 | export const processDeathFile = "test/Fly/BlueGreen.test.ts"; |
| 15 | export const phases = ["create", "promotion", "retirement"] as const; |
| 16 | export type Phase = (typeof phases)[number]; |
| 17 | |
| 18 | const Identity = Schema.Struct({ |
| 19 | id: Schema.String, |
| 20 | generation: Schema.String, |
| 21 | workload: Schema.String, |
| 22 | sequence: Schema.String, |
| 23 | digest: Schema.String, |
| 24 | instance: Schema.String, |
| 25 | fqn: Schema.String, |
| 26 | phase: Schema.String, |
| 27 | }); |
| 28 | |
| 29 | export const Witness = Schema.Struct({ |
| 30 | version: Schema.Literal(1), |
| 31 | signal: Schema.Literal("SIGKILL"), |
| 32 | phase: Schema.Literals(phases), |
| 33 | pid: Schema.Number, |
| 34 | cwd: Schema.String, |
| 35 | stack: Schema.String, |
| 36 | stage: Schema.String, |
| 37 | appName: Schema.String, |
| 38 | recordedAt: Schema.Number, |
| 39 | predecessor: Identity, |
| 40 | candidate: Identity, |
| 41 | barrier: Schema.Struct({ |
| 42 | sequence: Schema.Number, |
| 43 | method: Schema.String, |
| 44 | path: Schema.String, |
| 45 | status: Schema.Number, |
| 46 | machineId: Schema.String, |
| 47 | }), |
| 48 | leases: Schema.Array( |
| 49 | Schema.Struct({ |
| 50 | machineId: Schema.String, |
| 51 | expiresAt: Schema.Number, |
| 52 | }), |
| 53 | ), |
| 54 | row: Schema.Struct({ |
| 55 | fqn: Schema.String, |
| 56 | instanceId: Schema.String, |
| 57 | status: Schema.String, |
| 58 | }), |
| 59 | }); |
| 60 | export type Witness = typeof Witness.Type; |
| 61 | |
| 62 | // SDK transport failures can retain headers; never persist or print those objects. |
| 63 | const read = <A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) => |
| 64 | effect.pipe( |
| 65 | Retry.none, |
| 66 | Effect.timeout("10 seconds"), |
| 67 | Effect.mapError((error) => new Error(`Process-death SDK observation failed (${error._tag})`)), |
| 68 | ); |
| 69 | |
| 70 | export const nowSeconds = Effect.sync(() => Math.floor(Date.now() / 1000)); |
| 71 | |
| 72 | export const evidencePaths = (stack: ScratchStack) => |
| 73 | Effect.gen(function* () { |
| 74 | const path = yield* Path.Path; |
| 75 | const cwd = yield* Effect.sync(() => process.cwd()); |
| 76 | const directory = path.join(cwd, ".alchemy", "fly-process-death", stack.name, stack.stage); |
| 77 | return { |
| 78 | cwd, |
| 79 | directory, |
| 80 | witness: path.join(directory, "witness.json"), |
| 81 | finalized: path.join(directory, "finalized.json"), |
| 82 | recovered: path.join(directory, "recovered.json"), |
| 83 | }; |
| 84 | }); |
| 85 | |
| 86 | /** File and containing directory are synced before the caller can signal itself. */ |
| 87 | export const writeEvidence = (file: string, value: unknown) => |
| 88 | Effect.gen(function* () { |
| 89 | const fs = yield* FileSystem.FileSystem; |
| 90 | const path = yield* Path.Path; |
| 91 | const directory = path.dirname(file); |
| 92 | yield* fs.makeDirectory(directory, { recursive: true }); |
| 93 | const temporary = `${file}.tmp`; |
| 94 | const bytes = yield* Effect.sync(() => new TextEncoder().encode(JSON.stringify(value))); |
| 95 | yield* Effect.gen(function* () { |
| 96 | const handle = yield* fs.open(temporary, { flag: "w", mode: 0o600 }); |
| 97 | yield* handle.writeAll(bytes); |
| 98 | yield* handle.sync; |
| 99 | }).pipe(Effect.scoped); |
| 100 | yield* fs.rename(temporary, file); |
| 101 | yield* Effect.gen(function* () { |
| 102 | const handle = yield* fs.open(directory, { flag: "r" }); |
| 103 | yield* handle.sync; |
| 104 | }).pipe(Effect.scoped); |
| 105 | }); |
| 106 | |
| 107 | export const readWitness = (file: string) => |
| 108 | Effect.gen(function* () { |
| 109 | const fs = yield* FileSystem.FileSystem; |
| 110 | return yield* fs.readFileString(file).pipe( |
| 111 | Effect.flatMap( |
| 112 | Schema.decodeUnknownEffect(Schema.fromJsonString(Witness), { |
| 113 | onExcessProperty: "error", |
| 114 | }), |
| 115 | ), |
| 116 | Effect.mapError( |
| 117 | () => |
| 118 | new Error( |
| 119 | "Missing or invalid process-death witness; preserve state and inspect the crash leg", |
| 120 | ), |
| 121 | ), |
| 122 | ); |
| 123 | }); |
| 124 | |
| 125 | export const assertSingleRunner = Effect.gen(function* () { |
| 126 | const args = yield* Effect.sync(() => process.argv.slice(2)); |
| 127 | // pnpm test adds this exclusion before the explicitly focused file. |
| 128 | const focused = args[0] === "--exclude" && args[1] === "test/Railway" ? args.slice(2) : args; |
| 129 | expect(focused).toEqual([ |
| 130 | processDeathFile, |
| 131 | "-t", |
| 132 | " > process death > ", |
| 133 | "--profile", |
| 134 | "testing", |
| 135 | "--retry", |
| 136 | "0", |
| 137 | "--concurrency", |
| 138 | "1", |
| 139 | "--sequential", |
| 140 | "--timeout", |
| 141 | "500000", |
| 142 | ]); |
| 143 | expect(process.env.FLY_PROCESS_DEATH_ONLY_RUNNER).toBe("1"); |
| 144 | expect(process.env.ALCHEMY_PROFILE).toBe("testing"); |
| 145 | expect(process.env.ALCHEMY_TEST_DEV).toBe("0"); |
| 146 | expect(process.env.ALCHEMY_DEV).toBe("0"); |
| 147 | const fs = yield* FileSystem.FileSystem; |
| 148 | for (const file of [ |
| 149 | processDeathFile, |
| 150 | "test/Fly/fixtures/process-death.ts", |
| 151 | "test/Fly/fixtures/transport.ts", |
| 152 | ]) { |
| 153 | const source = yield* fs.readFileString(file); |
| 154 | expect(source).not.toMatch( |
| 155 | /(?:from\s+|import\s*(?:\(\s*)?|require\s*\()\s*["'](?:node:)?child_process["']/, |
| 156 | ); |
| 157 | expect(source).not.toMatch(/\b(?:Bun|Deno)\s*\.\s*(?:spawn|spawnSync|Command)\s*\(/); |
| 158 | expect(source).not.toMatch(/from\s+["'][^"']*(?:ChildProcess|CommandExecutor)[^"']*["']/); |
| 159 | } |
| 160 | }); |
| 161 | |
| 162 | export const deploy = (stack: ScratchStack, version: "one" | "two") => |
| 163 | stack.deploy( |
| 164 | Effect.gen(function* () { |
| 165 | const app = yield* Fly.App("Site"); |
| 166 | return yield* Fly.Machine("Worker", { |
| 167 | app, |
| 168 | region: "iad", |
| 169 | count: 1, |
| 170 | image: "nginx:alpine", |
| 171 | guest: { cpus: 1, memoryMb: 256 }, |
| 172 | env: { VERSION: version }, |
| 173 | checks: { |
| 174 | ready: { |
| 175 | type: "http", |
| 176 | port: 80, |
| 177 | path: "/", |
| 178 | interval: "2s", |
| 179 | timeout: "1s", |
| 180 | }, |
| 181 | }, |
| 182 | deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" }, |
| 183 | shutdown: { signal: "SIGQUIT", timeout: "5 seconds" }, |
| 184 | }); |
| 185 | }), |
| 186 | ); |
| 187 | |
| 188 | export const census = (appName: string) => |
| 189 | read(machines.listMachines({ app_name: appName })).pipe( |
| 190 | Effect.map((all) => all.filter((machine) => machine.state !== "destroyed")), |
| 191 | ); |
| 192 | |
| 193 | export const machine = (appName: string, machineId: string) => |
| 194 | read( |
| 195 | machines.getMachine({ |
| 196 | app_name: appName, |
| 197 | machine_id: machineId, |
| 198 | }), |
| 199 | ); |
| 200 | |
| 201 | export const identity = (value: machines.Machine, stack: ScratchStack) => |
| 202 | Effect.sync(() => { |
| 203 | const metadata = value.config?.metadata ?? {}; |
| 204 | expect(metadata["alchemy.stack"]).toBe(stack.name); |
| 205 | expect(metadata["alchemy.stage"]).toBe(stack.stage); |
| 206 | expect(metadata["alchemy.id"]).toBe("Worker"); |
| 207 | expect(metadata["alchemy.type"]).toBe("Fly.Machine"); |
| 208 | expect(metadata["alchemy.count"]).toBe("1"); |
| 209 | expect(metadata["alchemy.replica"]).toBe("0"); |
| 210 | expect(metadata["alchemy.deployment-protocol"]).toBe("1"); |
| 211 | const result = { |
| 212 | id: value.id!, |
| 213 | generation: metadata["alchemy.generation"]!, |
| 214 | workload: metadata["alchemy.workload"]!, |
| 215 | sequence: metadata["alchemy.sequence"]!, |
| 216 | digest: value.image_ref?.digest!, |
| 217 | instance: metadata["alchemy.instance"]!, |
| 218 | fqn: metadata["alchemy.fqn"]!, |
| 219 | phase: metadata["alchemy.phase"]!, |
| 220 | }; |
| 221 | for (const value of Object.values(result)) { |
| 222 | expect(typeof value).toBe("string"); |
| 223 | expect(value.length).toBeGreaterThan(0); |
| 224 | } |
| 225 | expect(result.digest).toMatch(/^sha256:/); |
| 226 | return result; |
| 227 | }); |
| 228 | |
| 229 | export const persistedRow = (stack: ScratchStack, fqn: string) => |
| 230 | Effect.gen(function* () { |
| 231 | const state = yield* yield* State; |
| 232 | expect(state.id).toBe("local"); |
| 233 | const row = yield* state.get({ |
| 234 | stack: stack.name, |
| 235 | stage: stack.stage, |
| 236 | fqn, |
| 237 | }); |
| 238 | if (!row || row.kind === "action") { |
| 239 | return yield* Effect.fail( |
| 240 | new Error("Expected the original durable resource row; refusing to recreate evidence"), |
| 241 | ); |
| 242 | } |
| 243 | return { fqn: row.fqn, instanceId: row.instanceId, status: row.status }; |
| 244 | }).pipe(Effect.provide(stack.state)); |
| 245 | |
| 246 | export const matchesBarrier = ( |
| 247 | phase: Phase, |
| 248 | appName: string, |
| 249 | predecessorId: string, |
| 250 | event: TransportEvent, |
| 251 | ) => { |
| 252 | const base = `/v1/apps/${appName}/machines`; |
| 253 | return phase === "create" |
| 254 | ? event.method === "POST" && event.path === base |
| 255 | : phase === "promotion" |
| 256 | ? event.method === "POST" && |
| 257 | event.path.startsWith(`${base}/`) && |
| 258 | event.path.endsWith("/uncordon") |
| 259 | : event.method === "DELETE" && event.path === `${base}/${predecessorId}`; |
| 260 | }; |
| 261 | |
| 262 | const lease = (appName: string, machineId: string) => |
| 263 | read( |
| 264 | machines |
| 265 | .getMachineLease({ |
| 266 | app_name: appName, |
| 267 | machine_id: machineId, |
| 268 | }) |
| 269 | .pipe( |
| 270 | Effect.map((value) => ({ |
| 271 | present: !!value.data?.nonce, |
| 272 | expiresAt: value.data?.expires_at ?? 0, |
| 273 | })), |
| 274 | Effect.catchTag("NotFound", () => Effect.succeed({ present: false, expiresAt: 0 })), |
| 275 | ), |
| 276 | ); |
| 277 | |
| 278 | export const heldLeases = (appName: string, ids: string[]) => |
| 279 | Effect.gen(function* () { |
| 280 | const leases = yield* Effect.forEach( |
| 281 | ids, |
| 282 | (machineId) => |
| 283 | lease(appName, machineId).pipe(Effect.map((value) => ({ machineId, ...value }))), |
| 284 | { concurrency: 2 }, |
| 285 | ); |
| 286 | const now = yield* nowSeconds; |
| 287 | const held = leases.filter((value) => value.present); |
| 288 | expect(held.length).toBeGreaterThan(0); |
| 289 | for (const value of held) { |
| 290 | expect(value.expiresAt).toBeGreaterThan(now + 20); |
| 291 | expect(value.expiresAt).toBeLessThanOrEqual(now + 130); |
| 292 | } |
| 293 | return held.map(({ machineId, expiresAt }) => ({ machineId, expiresAt })); |
| 294 | }); |
| 295 | |
| 296 | /** Observe live authority in the new process, then native expiry without release/acquire. */ |
| 297 | export const observeLeaseExpiry = (witness: Witness) => |
| 298 | Effect.gen(function* () { |
| 299 | expect(witness.leases.length).toBeGreaterThan(0); |
| 300 | const initial = yield* Effect.forEach( |
| 301 | witness.leases, |
| 302 | (held) => |
| 303 | Effect.gen(function* () { |
| 304 | const observed = yield* lease(witness.appName, held.machineId); |
| 305 | const now = yield* nowSeconds; |
| 306 | expect(observed.present).toBe(true); |
| 307 | expect(observed.expiresAt).toBeGreaterThan(now); |
| 308 | expect(observed.expiresAt).toBeGreaterThanOrEqual(held.expiresAt); |
| 309 | expect(observed.expiresAt).toBeLessThanOrEqual(witness.recordedAt + 130); |
| 310 | return { machineId: held.machineId, expiresAt: observed.expiresAt }; |
| 311 | }), |
| 312 | { concurrency: 2 }, |
| 313 | ); |
| 314 | const samples: { |
| 315 | machineId: string; |
| 316 | present: boolean; |
| 317 | expiresAt: number; |
| 318 | at: number; |
| 319 | }[] = []; |
| 320 | const expired = yield* Effect.forEach( |
| 321 | initial, |
| 322 | (held) => |
| 323 | Effect.gen(function* () { |
| 324 | const value = yield* lease(witness.appName, held.machineId); |
| 325 | const at = yield* nowSeconds; |
| 326 | yield* Effect.sync(() => samples.push({ machineId: held.machineId, ...value, at })); |
| 327 | if (value.present) expect(value.expiresAt).toBe(held.expiresAt); |
| 328 | if (!value.present) expect(at).toBeGreaterThanOrEqual(held.expiresAt); |
| 329 | return !value.present; |
| 330 | }), |
| 331 | { concurrency: 2 }, |
| 332 | ).pipe( |
| 333 | Effect.repeat({ |
| 334 | schedule: Schedule.spaced("3 seconds"), |
| 335 | times: 65, |
| 336 | until: (all) => all.every(Boolean), |
| 337 | }), |
| 338 | Effect.timeout("230 seconds"), |
| 339 | ); |
| 340 | expect(expired.every(Boolean)).toBe(true); |
| 341 | // A deleted Machine is not evidence of lease expiration. |
| 342 | for (const held of initial) { |
| 343 | const surviving = yield* machine(witness.appName, held.machineId); |
| 344 | expect(surviving.state).not.toBe("destroyed"); |
| 345 | } |
| 346 | return { initial, samples }; |
| 347 | }); |
| 348 | |
| 349 | export const assertBarrierInventory = (stack: ScratchStack, witness: Witness) => |
| 350 | Effect.gen(function* () { |
| 351 | const live = yield* census(witness.appName); |
| 352 | expect(live.map((value) => value.id).sort()).toEqual( |
| 353 | (witness.phase === "retirement" |
| 354 | ? [witness.candidate.id] |
| 355 | : [witness.predecessor.id, witness.candidate.id] |
| 356 | ).sort(), |
| 357 | ); |
| 358 | const candidate = yield* machine(witness.appName, witness.candidate.id); |
| 359 | expect(yield* identity(candidate, stack)).toEqual(witness.candidate); |
| 360 | expect(candidate.config?.env?.VERSION).toBe("two"); |
| 361 | expect(candidate.cordoned).toBe(witness.phase === "create"); |
| 362 | expect(witness.candidate.phase).toBe( |
| 363 | witness.phase === "create" |
| 364 | ? "candidate" |
| 365 | : witness.phase === "promotion" |
| 366 | ? "promoting" |
| 367 | : "active", |
| 368 | ); |
| 369 | if (witness.phase !== "retirement") { |
| 370 | const predecessor = yield* machine(witness.appName, witness.predecessor.id); |
| 371 | expect(yield* identity(predecessor, stack)).toEqual(witness.predecessor); |
| 372 | expect(predecessor.cordoned).toBe(false); |
| 373 | } |
| 374 | }); |
| 375 | |
| 376 | export const assertConverged = (stack: ScratchStack, witness: Witness, ids: string[]) => |
| 377 | Effect.gen(function* () { |
| 378 | expect(ids).toEqual([witness.candidate.id]); |
| 379 | const live = yield* census(witness.appName); |
| 380 | expect(live.map((value) => value.id)).toEqual(ids); |
| 381 | const current = yield* machine(witness.appName, witness.candidate.id).pipe( |
| 382 | Effect.repeat({ |
| 383 | schedule: Schedule.spaced("2 seconds"), |
| 384 | times: 20, |
| 385 | until: (value) => |
| 386 | value.state === "started" && |
| 387 | value.cordoned === false && |
| 388 | value.checks?.some((check) => check.name === "ready" && check.status === "passing") === |
| 389 | true, |
| 390 | }), |
| 391 | Effect.timeout("60 seconds"), |
| 392 | ); |
| 393 | expect(yield* identity(current, stack)).toEqual({ |
| 394 | ...witness.candidate, |
| 395 | phase: "active", |
| 396 | }); |
| 397 | expect(current.state).toBe("started"); |
| 398 | expect(current.cordoned).toBe(false); |
| 399 | expect(current.config?.env?.VERSION).toBe("two"); |
| 400 | const ready = current.checks?.filter((check) => check.name === "ready") ?? []; |
| 401 | expect(ready.length).toBeGreaterThan(0); |
| 402 | expect(ready.every((check) => check.status === "passing")).toBe(true); |
| 403 | expect(current.config?.metadata?.["alchemy.checked-instance"]).toBe(current.instance_id); |
| 404 | expect(current.instance_id).toBeDefined(); |
| 405 | expect(current.config?.metadata?.["alchemy.predecessors"]).toBe(witness.predecessor.id); |
| 406 | expect( |
| 407 | current.config?.metadata?.["alchemy.image"]?.endsWith(`@${witness.candidate.digest}`), |
| 408 | ).toBe(true); |
| 409 | const row = yield* persistedRow(stack, witness.row.fqn); |
| 410 | expect(row.instanceId).toBe(witness.row.instanceId); |
| 411 | expect(["created", "updated"]).toContain(row.status); |
| 412 | }); |
| 413 | |
| 414 | export const assertClean = (stack: ScratchStack, appName: string) => |
| 415 | Effect.gen(function* () { |
| 416 | const gone = yield* read( |
| 417 | machines.getApp({ app_name: appName }).pipe( |
| 418 | Effect.as(false), |
| 419 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 420 | ), |
| 421 | ).pipe( |
| 422 | Effect.repeat({ |
| 423 | schedule: Schedule.spaced("1 second"), |
| 424 | times: 8, |
| 425 | until: Boolean, |
| 426 | }), |
| 427 | ); |
| 428 | expect(gone).toBe(true); |
| 429 | const remaining = yield* read( |
| 430 | machines |
| 431 | .listMachines({ app_name: appName }) |
| 432 | .pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))), |
| 433 | ); |
| 434 | expect(remaining.filter((value) => value.state !== "destroyed")).toHaveLength(0); |
| 435 | const rows = yield* Effect.gen(function* () { |
| 436 | const state = yield* yield* State; |
| 437 | return yield* state.list({ stack: stack.name, stage: stack.stage }); |
| 438 | }).pipe(Effect.provide(stack.state)); |
| 439 | expect(rows).toHaveLength(0); |
| 440 | }); |