The best way to control Claude Code, Codex and more from any computer or phone

main
ts 440 lines 15.1 KB
Raw
1 import * as Fly from "@alchemy.run/fly";
2 import { expect } from "@alchemy.run/test";
3 import type { ScratchStack } from "@alchemy.run/test/Alchemy";
4 import * as machines from "@distilled.cloud/fly-io/machines";
5 import * as Retry from "@distilled.cloud/fly-io/Retry";
6 import { State } from "alchemy/State/State";
7 import * as Effect from "effect/Effect";
8 import * as FileSystem from "effect/FileSystem";
9 import * as Path from "effect/Path";
10 import * as Schedule from "effect/Schedule";
11 import * as Schema from "effect/Schema";
12 import type { TransportEvent } from "./transport.ts";
13
14 export const processDeathFile = "test/Fly/BlueGreen.test.ts";
15 export const phases = ["create", "promotion", "retirement"] as const;
16 export type Phase = (typeof phases)[number];
17
18 const Identity = Schema.Struct({
19 id: Schema.String,
20 generation: Schema.String,
21 workload: Schema.String,
22 sequence: Schema.String,
23 digest: Schema.String,
24 instance: Schema.String,
25 fqn: Schema.String,
26 phase: Schema.String,
27 });
28
29 export const Witness = Schema.Struct({
30 version: Schema.Literal(1),
31 signal: Schema.Literal("SIGKILL"),
32 phase: Schema.Literals(phases),
33 pid: Schema.Number,
34 cwd: Schema.String,
35 stack: Schema.String,
36 stage: Schema.String,
37 appName: Schema.String,
38 recordedAt: Schema.Number,
39 predecessor: Identity,
40 candidate: Identity,
41 barrier: Schema.Struct({
42 sequence: Schema.Number,
43 method: Schema.String,
44 path: Schema.String,
45 status: Schema.Number,
46 machineId: Schema.String,
47 }),
48 leases: Schema.Array(
49 Schema.Struct({
50 machineId: Schema.String,
51 expiresAt: Schema.Number,
52 }),
53 ),
54 row: Schema.Struct({
55 fqn: Schema.String,
56 instanceId: Schema.String,
57 status: Schema.String,
58 }),
59 });
60 export type Witness = typeof Witness.Type;
61
62 // SDK transport failures can retain headers; never persist or print those objects.
63 const read = <A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
64 effect.pipe(
65 Retry.none,
66 Effect.timeout("10 seconds"),
67 Effect.mapError((error) => new Error(`Process-death SDK observation failed (${error._tag})`)),
68 );
69
70 export const nowSeconds = Effect.sync(() => Math.floor(Date.now() / 1000));
71
72 export const evidencePaths = (stack: ScratchStack) =>
73 Effect.gen(function* () {
74 const path = yield* Path.Path;
75 const cwd = yield* Effect.sync(() => process.cwd());
76 const directory = path.join(cwd, ".alchemy", "fly-process-death", stack.name, stack.stage);
77 return {
78 cwd,
79 directory,
80 witness: path.join(directory, "witness.json"),
81 finalized: path.join(directory, "finalized.json"),
82 recovered: path.join(directory, "recovered.json"),
83 };
84 });
85
86 /** File and containing directory are synced before the caller can signal itself. */
87 export const writeEvidence = (file: string, value: unknown) =>
88 Effect.gen(function* () {
89 const fs = yield* FileSystem.FileSystem;
90 const path = yield* Path.Path;
91 const directory = path.dirname(file);
92 yield* fs.makeDirectory(directory, { recursive: true });
93 const temporary = `${file}.tmp`;
94 const bytes = yield* Effect.sync(() => new TextEncoder().encode(JSON.stringify(value)));
95 yield* Effect.gen(function* () {
96 const handle = yield* fs.open(temporary, { flag: "w", mode: 0o600 });
97 yield* handle.writeAll(bytes);
98 yield* handle.sync;
99 }).pipe(Effect.scoped);
100 yield* fs.rename(temporary, file);
101 yield* Effect.gen(function* () {
102 const handle = yield* fs.open(directory, { flag: "r" });
103 yield* handle.sync;
104 }).pipe(Effect.scoped);
105 });
106
107 export const readWitness = (file: string) =>
108 Effect.gen(function* () {
109 const fs = yield* FileSystem.FileSystem;
110 return yield* fs.readFileString(file).pipe(
111 Effect.flatMap(
112 Schema.decodeUnknownEffect(Schema.fromJsonString(Witness), {
113 onExcessProperty: "error",
114 }),
115 ),
116 Effect.mapError(
117 () =>
118 new Error(
119 "Missing or invalid process-death witness; preserve state and inspect the crash leg",
120 ),
121 ),
122 );
123 });
124
125 export const assertSingleRunner = Effect.gen(function* () {
126 const args = yield* Effect.sync(() => process.argv.slice(2));
127 // pnpm test adds this exclusion before the explicitly focused file.
128 const focused = args[0] === "--exclude" && args[1] === "test/Railway" ? args.slice(2) : args;
129 expect(focused).toEqual([
130 processDeathFile,
131 "-t",
132 " > process death > ",
133 "--profile",
134 "testing",
135 "--retry",
136 "0",
137 "--concurrency",
138 "1",
139 "--sequential",
140 "--timeout",
141 "500000",
142 ]);
143 expect(process.env.FLY_PROCESS_DEATH_ONLY_RUNNER).toBe("1");
144 expect(process.env.ALCHEMY_PROFILE).toBe("testing");
145 expect(process.env.ALCHEMY_TEST_DEV).toBe("0");
146 expect(process.env.ALCHEMY_DEV).toBe("0");
147 const fs = yield* FileSystem.FileSystem;
148 for (const file of [
149 processDeathFile,
150 "test/Fly/fixtures/process-death.ts",
151 "test/Fly/fixtures/transport.ts",
152 ]) {
153 const source = yield* fs.readFileString(file);
154 expect(source).not.toMatch(
155 /(?:from\s+|import\s*(?:\(\s*)?|require\s*\()\s*["'](?:node:)?child_process["']/,
156 );
157 expect(source).not.toMatch(/\b(?:Bun|Deno)\s*\.\s*(?:spawn|spawnSync|Command)\s*\(/);
158 expect(source).not.toMatch(/from\s+["'][^"']*(?:ChildProcess|CommandExecutor)[^"']*["']/);
159 }
160 });
161
162 export const deploy = (stack: ScratchStack, version: "one" | "two") =>
163 stack.deploy(
164 Effect.gen(function* () {
165 const app = yield* Fly.App("Site");
166 return yield* Fly.Machine("Worker", {
167 app,
168 region: "iad",
169 count: 1,
170 image: "nginx:alpine",
171 guest: { cpus: 1, memoryMb: 256 },
172 env: { VERSION: version },
173 checks: {
174 ready: {
175 type: "http",
176 port: 80,
177 path: "/",
178 interval: "2s",
179 timeout: "1s",
180 },
181 },
182 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
183 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
184 });
185 }),
186 );
187
188 export const census = (appName: string) =>
189 read(machines.listMachines({ app_name: appName })).pipe(
190 Effect.map((all) => all.filter((machine) => machine.state !== "destroyed")),
191 );
192
193 export const machine = (appName: string, machineId: string) =>
194 read(
195 machines.getMachine({
196 app_name: appName,
197 machine_id: machineId,
198 }),
199 );
200
201 export const identity = (value: machines.Machine, stack: ScratchStack) =>
202 Effect.sync(() => {
203 const metadata = value.config?.metadata ?? {};
204 expect(metadata["alchemy.stack"]).toBe(stack.name);
205 expect(metadata["alchemy.stage"]).toBe(stack.stage);
206 expect(metadata["alchemy.id"]).toBe("Worker");
207 expect(metadata["alchemy.type"]).toBe("Fly.Machine");
208 expect(metadata["alchemy.count"]).toBe("1");
209 expect(metadata["alchemy.replica"]).toBe("0");
210 expect(metadata["alchemy.deployment-protocol"]).toBe("1");
211 const result = {
212 id: value.id!,
213 generation: metadata["alchemy.generation"]!,
214 workload: metadata["alchemy.workload"]!,
215 sequence: metadata["alchemy.sequence"]!,
216 digest: value.image_ref?.digest!,
217 instance: metadata["alchemy.instance"]!,
218 fqn: metadata["alchemy.fqn"]!,
219 phase: metadata["alchemy.phase"]!,
220 };
221 for (const value of Object.values(result)) {
222 expect(typeof value).toBe("string");
223 expect(value.length).toBeGreaterThan(0);
224 }
225 expect(result.digest).toMatch(/^sha256:/);
226 return result;
227 });
228
229 export const persistedRow = (stack: ScratchStack, fqn: string) =>
230 Effect.gen(function* () {
231 const state = yield* yield* State;
232 expect(state.id).toBe("local");
233 const row = yield* state.get({
234 stack: stack.name,
235 stage: stack.stage,
236 fqn,
237 });
238 if (!row || row.kind === "action") {
239 return yield* Effect.fail(
240 new Error("Expected the original durable resource row; refusing to recreate evidence"),
241 );
242 }
243 return { fqn: row.fqn, instanceId: row.instanceId, status: row.status };
244 }).pipe(Effect.provide(stack.state));
245
246 export const matchesBarrier = (
247 phase: Phase,
248 appName: string,
249 predecessorId: string,
250 event: TransportEvent,
251 ) => {
252 const base = `/v1/apps/${appName}/machines`;
253 return phase === "create"
254 ? event.method === "POST" && event.path === base
255 : phase === "promotion"
256 ? event.method === "POST" &&
257 event.path.startsWith(`${base}/`) &&
258 event.path.endsWith("/uncordon")
259 : event.method === "DELETE" && event.path === `${base}/${predecessorId}`;
260 };
261
262 const lease = (appName: string, machineId: string) =>
263 read(
264 machines
265 .getMachineLease({
266 app_name: appName,
267 machine_id: machineId,
268 })
269 .pipe(
270 Effect.map((value) => ({
271 present: !!value.data?.nonce,
272 expiresAt: value.data?.expires_at ?? 0,
273 })),
274 Effect.catchTag("NotFound", () => Effect.succeed({ present: false, expiresAt: 0 })),
275 ),
276 );
277
278 export const heldLeases = (appName: string, ids: string[]) =>
279 Effect.gen(function* () {
280 const leases = yield* Effect.forEach(
281 ids,
282 (machineId) =>
283 lease(appName, machineId).pipe(Effect.map((value) => ({ machineId, ...value }))),
284 { concurrency: 2 },
285 );
286 const now = yield* nowSeconds;
287 const held = leases.filter((value) => value.present);
288 expect(held.length).toBeGreaterThan(0);
289 for (const value of held) {
290 expect(value.expiresAt).toBeGreaterThan(now + 20);
291 expect(value.expiresAt).toBeLessThanOrEqual(now + 130);
292 }
293 return held.map(({ machineId, expiresAt }) => ({ machineId, expiresAt }));
294 });
295
296 /** Observe live authority in the new process, then native expiry without release/acquire. */
297 export const observeLeaseExpiry = (witness: Witness) =>
298 Effect.gen(function* () {
299 expect(witness.leases.length).toBeGreaterThan(0);
300 const initial = yield* Effect.forEach(
301 witness.leases,
302 (held) =>
303 Effect.gen(function* () {
304 const observed = yield* lease(witness.appName, held.machineId);
305 const now = yield* nowSeconds;
306 expect(observed.present).toBe(true);
307 expect(observed.expiresAt).toBeGreaterThan(now);
308 expect(observed.expiresAt).toBeGreaterThanOrEqual(held.expiresAt);
309 expect(observed.expiresAt).toBeLessThanOrEqual(witness.recordedAt + 130);
310 return { machineId: held.machineId, expiresAt: observed.expiresAt };
311 }),
312 { concurrency: 2 },
313 );
314 const samples: {
315 machineId: string;
316 present: boolean;
317 expiresAt: number;
318 at: number;
319 }[] = [];
320 const expired = yield* Effect.forEach(
321 initial,
322 (held) =>
323 Effect.gen(function* () {
324 const value = yield* lease(witness.appName, held.machineId);
325 const at = yield* nowSeconds;
326 yield* Effect.sync(() => samples.push({ machineId: held.machineId, ...value, at }));
327 if (value.present) expect(value.expiresAt).toBe(held.expiresAt);
328 if (!value.present) expect(at).toBeGreaterThanOrEqual(held.expiresAt);
329 return !value.present;
330 }),
331 { concurrency: 2 },
332 ).pipe(
333 Effect.repeat({
334 schedule: Schedule.spaced("3 seconds"),
335 times: 65,
336 until: (all) => all.every(Boolean),
337 }),
338 Effect.timeout("230 seconds"),
339 );
340 expect(expired.every(Boolean)).toBe(true);
341 // A deleted Machine is not evidence of lease expiration.
342 for (const held of initial) {
343 const surviving = yield* machine(witness.appName, held.machineId);
344 expect(surviving.state).not.toBe("destroyed");
345 }
346 return { initial, samples };
347 });
348
349 export const assertBarrierInventory = (stack: ScratchStack, witness: Witness) =>
350 Effect.gen(function* () {
351 const live = yield* census(witness.appName);
352 expect(live.map((value) => value.id).sort()).toEqual(
353 (witness.phase === "retirement"
354 ? [witness.candidate.id]
355 : [witness.predecessor.id, witness.candidate.id]
356 ).sort(),
357 );
358 const candidate = yield* machine(witness.appName, witness.candidate.id);
359 expect(yield* identity(candidate, stack)).toEqual(witness.candidate);
360 expect(candidate.config?.env?.VERSION).toBe("two");
361 expect(candidate.cordoned).toBe(witness.phase === "create");
362 expect(witness.candidate.phase).toBe(
363 witness.phase === "create"
364 ? "candidate"
365 : witness.phase === "promotion"
366 ? "promoting"
367 : "active",
368 );
369 if (witness.phase !== "retirement") {
370 const predecessor = yield* machine(witness.appName, witness.predecessor.id);
371 expect(yield* identity(predecessor, stack)).toEqual(witness.predecessor);
372 expect(predecessor.cordoned).toBe(false);
373 }
374 });
375
376 export const assertConverged = (stack: ScratchStack, witness: Witness, ids: string[]) =>
377 Effect.gen(function* () {
378 expect(ids).toEqual([witness.candidate.id]);
379 const live = yield* census(witness.appName);
380 expect(live.map((value) => value.id)).toEqual(ids);
381 const current = yield* machine(witness.appName, witness.candidate.id).pipe(
382 Effect.repeat({
383 schedule: Schedule.spaced("2 seconds"),
384 times: 20,
385 until: (value) =>
386 value.state === "started" &&
387 value.cordoned === false &&
388 value.checks?.some((check) => check.name === "ready" && check.status === "passing") ===
389 true,
390 }),
391 Effect.timeout("60 seconds"),
392 );
393 expect(yield* identity(current, stack)).toEqual({
394 ...witness.candidate,
395 phase: "active",
396 });
397 expect(current.state).toBe("started");
398 expect(current.cordoned).toBe(false);
399 expect(current.config?.env?.VERSION).toBe("two");
400 const ready = current.checks?.filter((check) => check.name === "ready") ?? [];
401 expect(ready.length).toBeGreaterThan(0);
402 expect(ready.every((check) => check.status === "passing")).toBe(true);
403 expect(current.config?.metadata?.["alchemy.checked-instance"]).toBe(current.instance_id);
404 expect(current.instance_id).toBeDefined();
405 expect(current.config?.metadata?.["alchemy.predecessors"]).toBe(witness.predecessor.id);
406 expect(
407 current.config?.metadata?.["alchemy.image"]?.endsWith(`@${witness.candidate.digest}`),
408 ).toBe(true);
409 const row = yield* persistedRow(stack, witness.row.fqn);
410 expect(row.instanceId).toBe(witness.row.instanceId);
411 expect(["created", "updated"]).toContain(row.status);
412 });
413
414 export const assertClean = (stack: ScratchStack, appName: string) =>
415 Effect.gen(function* () {
416 const gone = yield* read(
417 machines.getApp({ app_name: appName }).pipe(
418 Effect.as(false),
419 Effect.catchTag("NotFound", () => Effect.succeed(true)),
420 ),
421 ).pipe(
422 Effect.repeat({
423 schedule: Schedule.spaced("1 second"),
424 times: 8,
425 until: Boolean,
426 }),
427 );
428 expect(gone).toBe(true);
429 const remaining = yield* read(
430 machines
431 .listMachines({ app_name: appName })
432 .pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
433 );
434 expect(remaining.filter((value) => value.state !== "destroyed")).toHaveLength(0);
435 const rows = yield* Effect.gen(function* () {
436 const state = yield* yield* State;
437 return yield* state.list({ stack: stack.name, stage: stack.stage });
438 }).pipe(Effect.provide(stack.state));
439 expect(rows).toHaveLength(0);
440 });