The best way to control Claude Code, Codex and more from any computer or phone

main
ts 410 lines 14.8 KB
Raw
1 import { randomBytes } from "node:crypto";
2 import * as Fly from "@alchemy.run/fly";
3 import { expect } from "@alchemy.run/test";
4 import type { ScratchStack } from "@alchemy.run/test/Alchemy";
5 import * as machines from "@distilled.cloud/fly-io/machines";
6 import * as Output from "alchemy/Output";
7 import * as Effect from "effect/Effect";
8 import { FileSystem } from "effect/FileSystem";
9 import * as HttpClient from "effect/http/HttpClient";
10 import * as HttpClientRequest from "effect/http/HttpClientRequest";
11 import * as Path from "effect/Path";
12 import * as Redacted from "effect/Redacted";
13 import * as Schedule from "effect/Schedule";
14 import { Ledger, ledgerLayer } from "./bluegreen-worker-ledger.ts";
15 import { Worker, workerLayer, type WorkerOptions } from "./bluegreen-worker-managed.ts";
16 import {
17 Cache,
18 LedgerSite,
19 WorkerSite,
20 services,
21 type LedgerEvent,
22 type Snapshot,
23 scripts,
24 } from "./bluegreen-worker-shared.ts";
25
26 const readinessProbe = `
27 const report = (value) => console.log(value);
28 report(process.env.REDIS_URL ? "redis-url:present" : "redis-url:missing");
29 report(process.env.LEDGER_TOKEN ? "ledger-token:present" : "ledger-token:missing");
30 report(["1", "2"].includes(process.env.WORKERS) ? "workers:valid" : "workers:missing-or-invalid");
31 report(["0", "1"].includes(process.env.RUN_ONLY) ? "run-only:valid" : "run-only:missing-or-invalid");
32 try {
33 const value = JSON.parse(process.env.WORKERS);
34 report(typeof value === "number" ? "workers-decoded:number" : "workers-decoded:other");
35 } catch { report("workers-decoded:other"); }
36 for (const [family, host] of [["ipv4", "127.0.0.1"], ["ipv6", "[::1]"]]) {
37 try {
38 const response = await fetch("http://" + host + ":3000/health", {
39 signal: AbortSignal.timeout(1500),
40 });
41 report(family + (response.status === 200 ? ":ready" : response.status === 503 ? ":starting" : ":other-status"));
42 await response.body?.cancel();
43 } catch { report(family + ":unreachable"); }
44 }
45 if (process.env.LEDGER_URL) {
46 try {
47 const response = await fetch(process.env.LEDGER_URL, {
48 method: "POST",
49 headers: { authorization: "Bearer " + process.env.LEDGER_TOKEN, "content-type": "application/json" },
50 body: JSON.stringify({ operation: "snapshot", args: [] }),
51 signal: AbortSignal.timeout(1500),
52 });
53 report(response.status === 200 ? "ledger:authenticated" : response.status === 401 ? "ledger:unauthorized" : "ledger:other-status");
54 await response.body?.cancel();
55 } catch { report("ledger:unreachable"); }
56 }
57 `;
58
59 const readinessProbeLabels = new Set([
60 "redis-url:present",
61 "redis-url:missing",
62 "ledger-token:present",
63 "ledger-token:missing",
64 "workers:valid",
65 "workers:missing-or-invalid",
66 "run-only:valid",
67 "run-only:missing-or-invalid",
68 "workers-decoded:number",
69 "workers-decoded:other",
70 "ipv4:ready",
71 "ipv4:starting",
72 "ipv4:other-status",
73 "ipv4:unreachable",
74 "ipv6:ready",
75 "ipv6:starting",
76 "ipv6:other-status",
77 "ipv6:unreachable",
78 "ledger:authenticated",
79 "ledger:unauthorized",
80 "ledger:other-status",
81 "ledger:unreachable",
82 ]);
83
84 const diagnoseMachines = (appName: string, role: "worker" | "ledger") =>
85 Effect.gen(function* () {
86 const live = yield* machines
87 .listMachines({ app_name: appName })
88 .pipe(Effect.timeout("8 seconds"));
89 yield* Effect.logError("Worker readiness machine census", {
90 role,
91 appName,
92 machines: live.map((machine) => ({
93 id: machine.id,
94 state: machine.state,
95 checks: machine.checks?.map((check) => ({
96 name: check.name,
97 status: check.status,
98 })),
99 })),
100 });
101 for (const machine of live.slice(0, 2)) {
102 const machineId = machine.id;
103 if (!machineId || machine.state !== "started") continue;
104 yield* machines
105 .execMachine({
106 app_name: appName,
107 machine_id: machineId,
108 command: ["node", "--input-type=module", "-e", readinessProbe],
109 timeout: 6,
110 })
111 .pipe(
112 Effect.timeout("8 seconds"),
113 Effect.flatMap((response) =>
114 Effect.logError("Worker readiness safe exec observations", {
115 role,
116 machineId,
117 exitCode: response.exit_code,
118 observations: (response.stdout ?? "")
119 .split("\n")
120 .filter((line) => readinessProbeLabels.has(line)),
121 }),
122 ),
123 Effect.catchCause(() =>
124 Effect.logWarning("Worker readiness exec unavailable", {
125 role,
126 machineId,
127 }),
128 ),
129 );
130 }
131 }).pipe(
132 Effect.catchCause(() =>
133 Effect.logWarning("Worker readiness machine census unavailable", {
134 role,
135 appName,
136 }),
137 ),
138 );
139
140 export const requireValue = <A>(value: A | undefined, name: string) =>
141 value === undefined ? Effect.fail(new Error(`Missing ${name}`)) : Effect.succeed(value);
142
143 export const makeScenario = (stack: ScratchStack) =>
144 Effect.gen(function* () {
145 const token = yield* Effect.sync(() =>
146 Array.from(randomBytes(32), (byte) => byte.toString(16).padStart(2, "0")).join(""),
147 );
148 const fs = yield* FileSystem;
149 const path = yield* Path.Path;
150 const raw = yield* fs.readFileString(
151 yield* path.fromFileUrl(new URL("./bluegreen-worker-raw.mjs", import.meta.url)),
152 );
153 const deploy = (
154 options: WorkerOptions & {
155 raw?: boolean;
156 rawSignal?: "SIGQUIT" | "SIGTERM";
157 gatewayOnly?: boolean;
158 },
159 ) =>
160 stack.deploy(
161 Effect.gen(function* () {
162 const cache = yield* Cache;
163 const ledgerApp = yield* LedgerSite;
164 const ledgerSecret = yield* Fly.Secret("LedgerToken", {
165 app: ledgerApp,
166 name: "LEDGER_TOKEN",
167 value: Redacted.make(token),
168 });
169 yield* Fly.IpAssignment("LedgerIp", {
170 app: ledgerApp,
171 type: "shared_v4",
172 });
173 const ledger = yield* Ledger.pipe(Effect.provide(ledgerLayer(ledgerSecret.digest)));
174 const ledgerUrl = ledger.url.pipe(
175 Output.mapEffect((url) =>
176 url ? Effect.succeed(url) : Effect.die(new Error("Ledger service URL missing")),
177 ),
178 );
179 const workerApp = yield* WorkerSite;
180 const workerSecret = yield* Fly.Secret("WorkerToken", {
181 app: workerApp,
182 name: "LEDGER_TOKEN",
183 value: Redacted.make(token),
184 });
185 if (!options.runOnly)
186 yield* Fly.IpAssignment("WorkerIp", {
187 app: workerApp,
188 type: "shared_v4",
189 });
190 const worker = options.gatewayOnly
191 ? undefined
192 : options.raw
193 ? yield* Fly.Machine("RawWorker", {
194 app: workerApp,
195 image: "node:26-slim",
196 region: "iad",
197 guest: { cpuKind: "shared", cpus: 1, memoryMb: 256 },
198 init: { exec: ["node", "--input-type=module", "-e", raw] },
199 env: {
200 VERSION: options.version,
201 MODE: options.mode ?? "drain",
202 AFTER_SIGNAL_MS: String(options.afterSignalMs ?? 1000),
203 SHUTDOWN_MS: String(
204 options.timeout === "60 seconds"
205 ? 60000
206 : options.timeout === "10 seconds"
207 ? 10000
208 : 30000,
209 ),
210 LEDGER_URL: ledgerUrl,
211 SECRET_READY: workerSecret.digest.pipe(
212 Output.mapEffect((digest) =>
213 requireValue(digest, "worker secret digest").pipe(Effect.orDie),
214 ),
215 ),
216 },
217 deploy: {
218 strategy: "bluegreen",
219 healthTimeout: "45 seconds",
220 },
221 shutdown: {
222 signal: options.rawSignal ?? options.signal,
223 timeout: options.timeout,
224 },
225 services: options.runOnly ? [] : services,
226 checks: options.runOnly
227 ? {
228 ready: {
229 type: "http",
230 port: 3000,
231 path: "/health",
232 interval: "2s",
233 timeout: "1s",
234 },
235 }
236 : undefined,
237 })
238 : yield* Worker.pipe(Effect.provide(workerLayer(options)));
239 return { cache, ledger, ledgerUrl, worker, workerApp, ledgerApp };
240 }),
241 );
242 type Deployment = Effect.Success<ReturnType<typeof deploy>>;
243 let inventory: Deployment | undefined;
244 const trackedDeploy = (options: Parameters<typeof deploy>[0]) =>
245 Effect.gen(function* () {
246 if (inventory === undefined) {
247 inventory = yield* deploy({ ...options, gatewayOnly: true });
248 if (options.gatewayOnly) return inventory;
249 }
250 const value = yield* deploy(options);
251 inventory = value;
252 return value;
253 }).pipe(
254 Effect.tapCause(() => {
255 const observed = inventory;
256 if (!observed) return Effect.void;
257 return Effect.all(
258 [
259 diagnoseMachines(observed.workerApp.appName, "worker"),
260 diagnoseMachines(observed.ledgerApp.appName, "ledger"),
261 snapshot(observed.ledgerUrl).pipe(
262 Effect.flatMap((value) =>
263 Effect.logError("Worker lifecycle ledger at failure", value.events.slice(-30)),
264 ),
265 Effect.catchCause(() =>
266 Effect.logWarning(
267 "Worker lifecycle ledger unavailable during failure diagnostics",
268 ),
269 ),
270 ),
271 ],
272 { concurrency: "unbounded", discard: true },
273 );
274 }),
275 );
276 const call = (
277 url: string,
278 operation: keyof typeof scripts,
279 args: string[] = [],
280 requestTimeout: "8 seconds" | "30 seconds" = "8 seconds",
281 ) =>
282 HttpClient.execute(
283 HttpClientRequest.post(url).pipe(
284 HttpClientRequest.setHeader("authorization", `Bearer ${token}`),
285 HttpClientRequest.bodyJsonUnsafe({ operation, args }),
286 ),
287 ).pipe(
288 Effect.flatMap((response) =>
289 response.status === 200
290 ? response.json
291 : Effect.fail(new Error(`ledger HTTP ${response.status}`)),
292 ),
293 Effect.map((body) => (body as { result: unknown }).result),
294 Effect.timeout(requestTimeout),
295 Effect.catchTag("TimeoutError", () =>
296 Effect.fail(new Error(`Ledger ${operation} request exceeded ${requestTimeout}`)),
297 ),
298 );
299 const snapshot = (url: string) =>
300 // Durable observations have a separate budget from the worker's shutdown grace.
301 call(url, "snapshot", [], "30 seconds").pipe(
302 Effect.map((result) => {
303 const value = JSON.parse(String(result)) as Snapshot;
304 return {
305 ...value,
306 events: (Array.isArray(value.events) ? value.events : []).map(
307 (event) => JSON.parse(event) as LedgerEvent,
308 ),
309 results: Array.isArray(value.results) ? value.results : [],
310 checkpoints: Array.isArray(value.checkpoints) ? value.checkpoints : [],
311 produced: Array.isArray(value.produced) ? value.produced : [],
312 };
313 }),
314 );
315 const wait = (
316 url: string,
317 predicate: (value: Effect.Success<ReturnType<typeof snapshot>>) => boolean,
318 ) =>
319 snapshot(url).pipe(
320 Effect.repeat({
321 until: predicate,
322 times: 10,
323 schedule: Schedule.spaced("1 second"),
324 }),
325 Effect.timeout("45 seconds"),
326 Effect.catchTag("TimeoutError", () =>
327 Effect.fail(new Error("Ledger predicate observation exceeded 45 seconds")),
328 ),
329 Effect.tap((value) => Effect.sync(() => expect(predicate(value)).toBe(true))),
330 );
331 const settle = (url: string) =>
332 Effect.gen(function* () {
333 yield* call(url, "pause");
334 const ledger = yield* wait(
335 url,
336 (value) => value.pending === 0 && value.results.length === value.produced.length * 2,
337 );
338 expect(ledger.entries).toBe(ledger.produced.length);
339 const completed = ledger.results.filter((_value, index) => index % 2 === 0);
340 expect([...completed].sort()).toEqual([...ledger.produced].sort());
341 return ledger;
342 });
343 const cleanup = Effect.gen(function* () {
344 yield* stack.destroy();
345 if (!inventory) return;
346 for (const appName of [inventory.workerApp.appName, inventory.ledgerApp.appName]) {
347 const gone = yield* machines.getApp({ app_name: appName }).pipe(
348 Effect.as(false),
349 Effect.catchTag("NotFound", () => Effect.succeed(true)),
350 Effect.repeat({
351 until: (value) => value,
352 times: 8,
353 schedule: Schedule.spaced("1 second"),
354 }),
355 );
356 expect(gone).toBe(true);
357 }
358 expect(
359 yield* Fly.findRedisAddOn({
360 id: inventory.cache.redisId,
361 name: inventory.cache.name,
362 }),
363 ).toBeUndefined();
364 }).pipe(Effect.orDie);
365 return { deploy: trackedDeploy, call, snapshot, wait, settle, cleanup };
366 });
367
368 export const assertOrder = (
369 events: LedgerEvent[],
370 machine: string,
371 first: string,
372 second: string,
373 worker?: string,
374 ) => {
375 const selected = events.filter(
376 (event) => event.machine === machine && (worker === undefined || event.worker === worker),
377 );
378 expect(selected.findIndex((event) => event.event === first)).toBeGreaterThanOrEqual(0);
379 expect(selected.findIndex((event) => event.event === second)).toBeGreaterThan(
380 selected.findIndex((event) => event.event === first),
381 );
382 };
383 export const assertStopped = (events: LedgerEvent[], machine: string) => {
384 const stopped = events.findLastIndex(
385 (event) => event.machine === machine && event.event === "stopped",
386 );
387 expect(stopped).toBeGreaterThanOrEqual(0);
388 expect(
389 events
390 .slice(stopped + 1)
391 .filter(
392 (event) =>
393 event.machine === machine && ["claimed", "reclaimed", "producer"].includes(event.event),
394 ),
395 ).toEqual([]);
396 };
397 export const assertReplacement = (appName: string, oldId: string, newId: string) =>
398 Effect.gen(function* () {
399 expect(newId).not.toBe(oldId);
400 const live = (yield* machines.listMachines({ app_name: appName })).filter(
401 (machine) => machine.state !== "destroyed",
402 );
403 expect(live.map((machine) => machine.id)).toEqual([newId]);
404 expect(
405 yield* machines.getMachine({ app_name: appName, machine_id: oldId }).pipe(
406 Effect.map((machine) => machine.state === "destroyed"),
407 Effect.catchTag("NotFound", () => Effect.succeed(true)),
408 ),
409 ).toBe(true);
410 });