The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { randomBytes } from "node:crypto"; |
| 2 | import * as Fly from "@alchemy.run/fly"; |
| 3 | import { expect } from "@alchemy.run/test"; |
| 4 | import type { ScratchStack } from "@alchemy.run/test/Alchemy"; |
| 5 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 6 | import * as Output from "alchemy/Output"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import { FileSystem } from "effect/FileSystem"; |
| 9 | import * as HttpClient from "effect/http/HttpClient"; |
| 10 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 11 | import * as Path from "effect/Path"; |
| 12 | import * as Redacted from "effect/Redacted"; |
| 13 | import * as Schedule from "effect/Schedule"; |
| 14 | import { Ledger, ledgerLayer } from "./bluegreen-worker-ledger.ts"; |
| 15 | import { Worker, workerLayer, type WorkerOptions } from "./bluegreen-worker-managed.ts"; |
| 16 | import { |
| 17 | Cache, |
| 18 | LedgerSite, |
| 19 | WorkerSite, |
| 20 | services, |
| 21 | type LedgerEvent, |
| 22 | type Snapshot, |
| 23 | scripts, |
| 24 | } from "./bluegreen-worker-shared.ts"; |
| 25 | |
| 26 | const readinessProbe = ` |
| 27 | const report = (value) => console.log(value); |
| 28 | report(process.env.REDIS_URL ? "redis-url:present" : "redis-url:missing"); |
| 29 | report(process.env.LEDGER_TOKEN ? "ledger-token:present" : "ledger-token:missing"); |
| 30 | report(["1", "2"].includes(process.env.WORKERS) ? "workers:valid" : "workers:missing-or-invalid"); |
| 31 | report(["0", "1"].includes(process.env.RUN_ONLY) ? "run-only:valid" : "run-only:missing-or-invalid"); |
| 32 | try { |
| 33 | const value = JSON.parse(process.env.WORKERS); |
| 34 | report(typeof value === "number" ? "workers-decoded:number" : "workers-decoded:other"); |
| 35 | } catch { report("workers-decoded:other"); } |
| 36 | for (const [family, host] of [["ipv4", "127.0.0.1"], ["ipv6", "[::1]"]]) { |
| 37 | try { |
| 38 | const response = await fetch("http://" + host + ":3000/health", { |
| 39 | signal: AbortSignal.timeout(1500), |
| 40 | }); |
| 41 | report(family + (response.status === 200 ? ":ready" : response.status === 503 ? ":starting" : ":other-status")); |
| 42 | await response.body?.cancel(); |
| 43 | } catch { report(family + ":unreachable"); } |
| 44 | } |
| 45 | if (process.env.LEDGER_URL) { |
| 46 | try { |
| 47 | const response = await fetch(process.env.LEDGER_URL, { |
| 48 | method: "POST", |
| 49 | headers: { authorization: "Bearer " + process.env.LEDGER_TOKEN, "content-type": "application/json" }, |
| 50 | body: JSON.stringify({ operation: "snapshot", args: [] }), |
| 51 | signal: AbortSignal.timeout(1500), |
| 52 | }); |
| 53 | report(response.status === 200 ? "ledger:authenticated" : response.status === 401 ? "ledger:unauthorized" : "ledger:other-status"); |
| 54 | await response.body?.cancel(); |
| 55 | } catch { report("ledger:unreachable"); } |
| 56 | } |
| 57 | `; |
| 58 | |
| 59 | const readinessProbeLabels = new Set([ |
| 60 | "redis-url:present", |
| 61 | "redis-url:missing", |
| 62 | "ledger-token:present", |
| 63 | "ledger-token:missing", |
| 64 | "workers:valid", |
| 65 | "workers:missing-or-invalid", |
| 66 | "run-only:valid", |
| 67 | "run-only:missing-or-invalid", |
| 68 | "workers-decoded:number", |
| 69 | "workers-decoded:other", |
| 70 | "ipv4:ready", |
| 71 | "ipv4:starting", |
| 72 | "ipv4:other-status", |
| 73 | "ipv4:unreachable", |
| 74 | "ipv6:ready", |
| 75 | "ipv6:starting", |
| 76 | "ipv6:other-status", |
| 77 | "ipv6:unreachable", |
| 78 | "ledger:authenticated", |
| 79 | "ledger:unauthorized", |
| 80 | "ledger:other-status", |
| 81 | "ledger:unreachable", |
| 82 | ]); |
| 83 | |
| 84 | const diagnoseMachines = (appName: string, role: "worker" | "ledger") => |
| 85 | Effect.gen(function* () { |
| 86 | const live = yield* machines |
| 87 | .listMachines({ app_name: appName }) |
| 88 | .pipe(Effect.timeout("8 seconds")); |
| 89 | yield* Effect.logError("Worker readiness machine census", { |
| 90 | role, |
| 91 | appName, |
| 92 | machines: live.map((machine) => ({ |
| 93 | id: machine.id, |
| 94 | state: machine.state, |
| 95 | checks: machine.checks?.map((check) => ({ |
| 96 | name: check.name, |
| 97 | status: check.status, |
| 98 | })), |
| 99 | })), |
| 100 | }); |
| 101 | for (const machine of live.slice(0, 2)) { |
| 102 | const machineId = machine.id; |
| 103 | if (!machineId || machine.state !== "started") continue; |
| 104 | yield* machines |
| 105 | .execMachine({ |
| 106 | app_name: appName, |
| 107 | machine_id: machineId, |
| 108 | command: ["node", "--input-type=module", "-e", readinessProbe], |
| 109 | timeout: 6, |
| 110 | }) |
| 111 | .pipe( |
| 112 | Effect.timeout("8 seconds"), |
| 113 | Effect.flatMap((response) => |
| 114 | Effect.logError("Worker readiness safe exec observations", { |
| 115 | role, |
| 116 | machineId, |
| 117 | exitCode: response.exit_code, |
| 118 | observations: (response.stdout ?? "") |
| 119 | .split("\n") |
| 120 | .filter((line) => readinessProbeLabels.has(line)), |
| 121 | }), |
| 122 | ), |
| 123 | Effect.catchCause(() => |
| 124 | Effect.logWarning("Worker readiness exec unavailable", { |
| 125 | role, |
| 126 | machineId, |
| 127 | }), |
| 128 | ), |
| 129 | ); |
| 130 | } |
| 131 | }).pipe( |
| 132 | Effect.catchCause(() => |
| 133 | Effect.logWarning("Worker readiness machine census unavailable", { |
| 134 | role, |
| 135 | appName, |
| 136 | }), |
| 137 | ), |
| 138 | ); |
| 139 | |
| 140 | export const requireValue = <A>(value: A | undefined, name: string) => |
| 141 | value === undefined ? Effect.fail(new Error(`Missing ${name}`)) : Effect.succeed(value); |
| 142 | |
| 143 | export const makeScenario = (stack: ScratchStack) => |
| 144 | Effect.gen(function* () { |
| 145 | const token = yield* Effect.sync(() => |
| 146 | Array.from(randomBytes(32), (byte) => byte.toString(16).padStart(2, "0")).join(""), |
| 147 | ); |
| 148 | const fs = yield* FileSystem; |
| 149 | const path = yield* Path.Path; |
| 150 | const raw = yield* fs.readFileString( |
| 151 | yield* path.fromFileUrl(new URL("./bluegreen-worker-raw.mjs", import.meta.url)), |
| 152 | ); |
| 153 | const deploy = ( |
| 154 | options: WorkerOptions & { |
| 155 | raw?: boolean; |
| 156 | rawSignal?: "SIGQUIT" | "SIGTERM"; |
| 157 | gatewayOnly?: boolean; |
| 158 | }, |
| 159 | ) => |
| 160 | stack.deploy( |
| 161 | Effect.gen(function* () { |
| 162 | const cache = yield* Cache; |
| 163 | const ledgerApp = yield* LedgerSite; |
| 164 | const ledgerSecret = yield* Fly.Secret("LedgerToken", { |
| 165 | app: ledgerApp, |
| 166 | name: "LEDGER_TOKEN", |
| 167 | value: Redacted.make(token), |
| 168 | }); |
| 169 | yield* Fly.IpAssignment("LedgerIp", { |
| 170 | app: ledgerApp, |
| 171 | type: "shared_v4", |
| 172 | }); |
| 173 | const ledger = yield* Ledger.pipe(Effect.provide(ledgerLayer(ledgerSecret.digest))); |
| 174 | const ledgerUrl = ledger.url.pipe( |
| 175 | Output.mapEffect((url) => |
| 176 | url ? Effect.succeed(url) : Effect.die(new Error("Ledger service URL missing")), |
| 177 | ), |
| 178 | ); |
| 179 | const workerApp = yield* WorkerSite; |
| 180 | const workerSecret = yield* Fly.Secret("WorkerToken", { |
| 181 | app: workerApp, |
| 182 | name: "LEDGER_TOKEN", |
| 183 | value: Redacted.make(token), |
| 184 | }); |
| 185 | if (!options.runOnly) |
| 186 | yield* Fly.IpAssignment("WorkerIp", { |
| 187 | app: workerApp, |
| 188 | type: "shared_v4", |
| 189 | }); |
| 190 | const worker = options.gatewayOnly |
| 191 | ? undefined |
| 192 | : options.raw |
| 193 | ? yield* Fly.Machine("RawWorker", { |
| 194 | app: workerApp, |
| 195 | image: "node:26-slim", |
| 196 | region: "iad", |
| 197 | guest: { cpuKind: "shared", cpus: 1, memoryMb: 256 }, |
| 198 | init: { exec: ["node", "--input-type=module", "-e", raw] }, |
| 199 | env: { |
| 200 | VERSION: options.version, |
| 201 | MODE: options.mode ?? "drain", |
| 202 | AFTER_SIGNAL_MS: String(options.afterSignalMs ?? 1000), |
| 203 | SHUTDOWN_MS: String( |
| 204 | options.timeout === "60 seconds" |
| 205 | ? 60000 |
| 206 | : options.timeout === "10 seconds" |
| 207 | ? 10000 |
| 208 | : 30000, |
| 209 | ), |
| 210 | LEDGER_URL: ledgerUrl, |
| 211 | SECRET_READY: workerSecret.digest.pipe( |
| 212 | Output.mapEffect((digest) => |
| 213 | requireValue(digest, "worker secret digest").pipe(Effect.orDie), |
| 214 | ), |
| 215 | ), |
| 216 | }, |
| 217 | deploy: { |
| 218 | strategy: "bluegreen", |
| 219 | healthTimeout: "45 seconds", |
| 220 | }, |
| 221 | shutdown: { |
| 222 | signal: options.rawSignal ?? options.signal, |
| 223 | timeout: options.timeout, |
| 224 | }, |
| 225 | services: options.runOnly ? [] : services, |
| 226 | checks: options.runOnly |
| 227 | ? { |
| 228 | ready: { |
| 229 | type: "http", |
| 230 | port: 3000, |
| 231 | path: "/health", |
| 232 | interval: "2s", |
| 233 | timeout: "1s", |
| 234 | }, |
| 235 | } |
| 236 | : undefined, |
| 237 | }) |
| 238 | : yield* Worker.pipe(Effect.provide(workerLayer(options))); |
| 239 | return { cache, ledger, ledgerUrl, worker, workerApp, ledgerApp }; |
| 240 | }), |
| 241 | ); |
| 242 | type Deployment = Effect.Success<ReturnType<typeof deploy>>; |
| 243 | let inventory: Deployment | undefined; |
| 244 | const trackedDeploy = (options: Parameters<typeof deploy>[0]) => |
| 245 | Effect.gen(function* () { |
| 246 | if (inventory === undefined) { |
| 247 | inventory = yield* deploy({ ...options, gatewayOnly: true }); |
| 248 | if (options.gatewayOnly) return inventory; |
| 249 | } |
| 250 | const value = yield* deploy(options); |
| 251 | inventory = value; |
| 252 | return value; |
| 253 | }).pipe( |
| 254 | Effect.tapCause(() => { |
| 255 | const observed = inventory; |
| 256 | if (!observed) return Effect.void; |
| 257 | return Effect.all( |
| 258 | [ |
| 259 | diagnoseMachines(observed.workerApp.appName, "worker"), |
| 260 | diagnoseMachines(observed.ledgerApp.appName, "ledger"), |
| 261 | snapshot(observed.ledgerUrl).pipe( |
| 262 | Effect.flatMap((value) => |
| 263 | Effect.logError("Worker lifecycle ledger at failure", value.events.slice(-30)), |
| 264 | ), |
| 265 | Effect.catchCause(() => |
| 266 | Effect.logWarning( |
| 267 | "Worker lifecycle ledger unavailable during failure diagnostics", |
| 268 | ), |
| 269 | ), |
| 270 | ), |
| 271 | ], |
| 272 | { concurrency: "unbounded", discard: true }, |
| 273 | ); |
| 274 | }), |
| 275 | ); |
| 276 | const call = ( |
| 277 | url: string, |
| 278 | operation: keyof typeof scripts, |
| 279 | args: string[] = [], |
| 280 | requestTimeout: "8 seconds" | "30 seconds" = "8 seconds", |
| 281 | ) => |
| 282 | HttpClient.execute( |
| 283 | HttpClientRequest.post(url).pipe( |
| 284 | HttpClientRequest.setHeader("authorization", `Bearer ${token}`), |
| 285 | HttpClientRequest.bodyJsonUnsafe({ operation, args }), |
| 286 | ), |
| 287 | ).pipe( |
| 288 | Effect.flatMap((response) => |
| 289 | response.status === 200 |
| 290 | ? response.json |
| 291 | : Effect.fail(new Error(`ledger HTTP ${response.status}`)), |
| 292 | ), |
| 293 | Effect.map((body) => (body as { result: unknown }).result), |
| 294 | Effect.timeout(requestTimeout), |
| 295 | Effect.catchTag("TimeoutError", () => |
| 296 | Effect.fail(new Error(`Ledger ${operation} request exceeded ${requestTimeout}`)), |
| 297 | ), |
| 298 | ); |
| 299 | const snapshot = (url: string) => |
| 300 | // Durable observations have a separate budget from the worker's shutdown grace. |
| 301 | call(url, "snapshot", [], "30 seconds").pipe( |
| 302 | Effect.map((result) => { |
| 303 | const value = JSON.parse(String(result)) as Snapshot; |
| 304 | return { |
| 305 | ...value, |
| 306 | events: (Array.isArray(value.events) ? value.events : []).map( |
| 307 | (event) => JSON.parse(event) as LedgerEvent, |
| 308 | ), |
| 309 | results: Array.isArray(value.results) ? value.results : [], |
| 310 | checkpoints: Array.isArray(value.checkpoints) ? value.checkpoints : [], |
| 311 | produced: Array.isArray(value.produced) ? value.produced : [], |
| 312 | }; |
| 313 | }), |
| 314 | ); |
| 315 | const wait = ( |
| 316 | url: string, |
| 317 | predicate: (value: Effect.Success<ReturnType<typeof snapshot>>) => boolean, |
| 318 | ) => |
| 319 | snapshot(url).pipe( |
| 320 | Effect.repeat({ |
| 321 | until: predicate, |
| 322 | times: 10, |
| 323 | schedule: Schedule.spaced("1 second"), |
| 324 | }), |
| 325 | Effect.timeout("45 seconds"), |
| 326 | Effect.catchTag("TimeoutError", () => |
| 327 | Effect.fail(new Error("Ledger predicate observation exceeded 45 seconds")), |
| 328 | ), |
| 329 | Effect.tap((value) => Effect.sync(() => expect(predicate(value)).toBe(true))), |
| 330 | ); |
| 331 | const settle = (url: string) => |
| 332 | Effect.gen(function* () { |
| 333 | yield* call(url, "pause"); |
| 334 | const ledger = yield* wait( |
| 335 | url, |
| 336 | (value) => value.pending === 0 && value.results.length === value.produced.length * 2, |
| 337 | ); |
| 338 | expect(ledger.entries).toBe(ledger.produced.length); |
| 339 | const completed = ledger.results.filter((_value, index) => index % 2 === 0); |
| 340 | expect([...completed].sort()).toEqual([...ledger.produced].sort()); |
| 341 | return ledger; |
| 342 | }); |
| 343 | const cleanup = Effect.gen(function* () { |
| 344 | yield* stack.destroy(); |
| 345 | if (!inventory) return; |
| 346 | for (const appName of [inventory.workerApp.appName, inventory.ledgerApp.appName]) { |
| 347 | const gone = yield* machines.getApp({ app_name: appName }).pipe( |
| 348 | Effect.as(false), |
| 349 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 350 | Effect.repeat({ |
| 351 | until: (value) => value, |
| 352 | times: 8, |
| 353 | schedule: Schedule.spaced("1 second"), |
| 354 | }), |
| 355 | ); |
| 356 | expect(gone).toBe(true); |
| 357 | } |
| 358 | expect( |
| 359 | yield* Fly.findRedisAddOn({ |
| 360 | id: inventory.cache.redisId, |
| 361 | name: inventory.cache.name, |
| 362 | }), |
| 363 | ).toBeUndefined(); |
| 364 | }).pipe(Effect.orDie); |
| 365 | return { deploy: trackedDeploy, call, snapshot, wait, settle, cleanup }; |
| 366 | }); |
| 367 | |
| 368 | export const assertOrder = ( |
| 369 | events: LedgerEvent[], |
| 370 | machine: string, |
| 371 | first: string, |
| 372 | second: string, |
| 373 | worker?: string, |
| 374 | ) => { |
| 375 | const selected = events.filter( |
| 376 | (event) => event.machine === machine && (worker === undefined || event.worker === worker), |
| 377 | ); |
| 378 | expect(selected.findIndex((event) => event.event === first)).toBeGreaterThanOrEqual(0); |
| 379 | expect(selected.findIndex((event) => event.event === second)).toBeGreaterThan( |
| 380 | selected.findIndex((event) => event.event === first), |
| 381 | ); |
| 382 | }; |
| 383 | export const assertStopped = (events: LedgerEvent[], machine: string) => { |
| 384 | const stopped = events.findLastIndex( |
| 385 | (event) => event.machine === machine && event.event === "stopped", |
| 386 | ); |
| 387 | expect(stopped).toBeGreaterThanOrEqual(0); |
| 388 | expect( |
| 389 | events |
| 390 | .slice(stopped + 1) |
| 391 | .filter( |
| 392 | (event) => |
| 393 | event.machine === machine && ["claimed", "reclaimed", "producer"].includes(event.event), |
| 394 | ), |
| 395 | ).toEqual([]); |
| 396 | }; |
| 397 | export const assertReplacement = (appName: string, oldId: string, newId: string) => |
| 398 | Effect.gen(function* () { |
| 399 | expect(newId).not.toBe(oldId); |
| 400 | const live = (yield* machines.listMachines({ app_name: appName })).filter( |
| 401 | (machine) => machine.state !== "destroyed", |
| 402 | ); |
| 403 | expect(live.map((machine) => machine.id)).toEqual([newId]); |
| 404 | expect( |
| 405 | yield* machines.getMachine({ app_name: appName, machine_id: oldId }).pipe( |
| 406 | Effect.map((machine) => machine.state === "destroyed"), |
| 407 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 408 | ), |
| 409 | ).toBe(true); |
| 410 | }); |