The best way to control Claude Code, Codex and more from any computer or phone

main
ts 383 lines 14.4 KB
Raw
1 import { expect } from "@alchemy.run/test";
2 import type { ScratchStack } from "@alchemy.run/test/Alchemy";
3 import * as machines from "@distilled.cloud/fly-io/machines";
4 import * as Retry from "@distilled.cloud/fly-io/Retry";
5 import * as Effect from "effect/Effect";
6 import * as FileSystem from "effect/FileSystem";
7 import * as Path from "effect/Path";
8 import * as Schedule from "effect/Schedule";
9 import * as Schema from "effect/Schema";
10 import {
11 Witness as ProcessDeathWitness,
12 census,
13 evidencePaths,
14 identity,
15 machine,
16 nowSeconds,
17 persistedRow,
18 } from "./process-death.ts";
19 import type { TransportEvent } from "./transport.ts";
20
21 export const signalOverlapFile = "test/Fly/BlueGreen.test.ts";
22 export const cases = [
23 { name: "sigint-create", signal: "SIGINT", phase: "create" },
24 { name: "sigint-promotion", signal: "SIGINT", phase: "promotion" },
25 { name: "sigint-overlap", signal: "SIGINT", phase: "overlap" },
26 { name: "sigint-retirement", signal: "SIGINT", phase: "retirement" },
27 { name: "sigkill-overlap", signal: "SIGKILL", phase: "overlap" },
28 ] as const;
29 export type SignalCase = (typeof cases)[number];
30
31 const Boundary = Schema.Struct({
32 sequence: Schema.Number,
33 method: Schema.String,
34 path: Schema.String,
35 status: Schema.Number,
36 machineId: Schema.String,
37 });
38
39 export const Witness = Schema.Struct({
40 ...ProcessDeathWitness.fields,
41 signal: Schema.Literals(["SIGINT", "SIGKILL"]),
42 phase: Schema.Literals(["create", "promotion", "overlap", "retirement"]),
43 case: Schema.String,
44 barrier: Boundary,
45 // The subsequent routing read proves the provider consumed the uncordon response.
46 returnedUncordon: Schema.optional(Boundary),
47 });
48 export type Witness = typeof Witness.Type;
49
50 const LeaseObservation = Schema.Struct({
51 machineId: Schema.String,
52 present: Schema.Boolean,
53 expiresAt: Schema.Number,
54 at: Schema.Number,
55 });
56
57 export const Finalized = Schema.Struct({
58 pid: Schema.Number,
59 case: Schema.String,
60 witnessRecordedAt: Schema.Number,
61 interruptedOnly: Schema.Boolean,
62 at: Schema.Number,
63 releases: Schema.Array(Boundary),
64 leases: Schema.Array(LeaseObservation),
65 });
66
67 export const RunnerInterrupted = Schema.Struct({
68 pid: Schema.Number,
69 case: Schema.String,
70 signal: Schema.Literal("SIGINT"),
71 witnessRecordedAt: Schema.Number,
72 at: Schema.Number,
73 deployFinalized: Schema.Boolean,
74 });
75
76 export const pathsFor = (stack: ScratchStack) =>
77 Effect.gen(function* () {
78 const paths = yield* evidencePaths(stack);
79 const path = yield* Path.Path;
80 return {
81 ...paths,
82 runnerInterrupted: path.join(paths.directory, "runner-interrupted.json"),
83 blocked: path.join(paths.directory, "interruption-limit.json"),
84 };
85 });
86
87 export const readEvidence = <S extends Schema.Top>(file: string, schema: S) =>
88 Effect.gen(function* () {
89 const fs = yield* FileSystem.FileSystem;
90 return yield* fs.readFileString(file).pipe(
91 Effect.flatMap(
92 Schema.decodeUnknownEffect(Schema.fromJsonString(schema), {
93 onExcessProperty: "error",
94 }),
95 ),
96 Effect.mapError(() => new Error("Missing or invalid signal evidence; preserve the attempt")),
97 );
98 });
99
100 export const assertSingleRunner = Effect.gen(function* () {
101 const args = yield* Effect.sync(() => process.argv.slice(2));
102 const focused = args[0] === "--exclude" && args[1] === "test/Railway" ? args.slice(2) : args;
103 expect(focused).toEqual([
104 signalOverlapFile,
105 "-t",
106 " > process signals and overlap > ",
107 "--profile",
108 "testing",
109 "--retry",
110 "0",
111 "--concurrency",
112 "1",
113 "--sequential",
114 "--timeout",
115 "750000",
116 ]);
117 expect(process.env.FLY_SIGNAL_OVERLAP_ONLY_RUNNER).toBe("1");
118 expect(process.env.FLY_PROCESS_DEATH_CASE).toBeUndefined();
119 expect(process.env.FLY_PROCESS_DEATH_MODE).toBeUndefined();
120 expect(process.env.ALCHEMY_PROFILE).toBe("testing");
121 expect(process.env.ALCHEMY_TEST_DEV).toBe("0");
122 expect(process.env.ALCHEMY_DEV).toBe("0");
123 expect(process.env.ALCHEMY_TEST_STAGE).toBe("test_fly_signal_overlap");
124 expect(process.env.FAST).toBeUndefined();
125 expect(process.versions.bun).toBeDefined();
126 const fs = yield* FileSystem.FileSystem;
127 for (const file of [
128 signalOverlapFile,
129 "test/Fly/fixtures/signal-overlap.ts",
130 "test/Fly/fixtures/process-death.ts",
131 "test/Fly/fixtures/transport.ts",
132 ]) {
133 const source = yield* fs.readFileString(file);
134 expect(source).not.toMatch(
135 /(?:from\s+|import\s*(?:\(\s*)?|require\s*\()\s*["'](?:node:)?child_process["']/,
136 );
137 expect(source).not.toMatch(/\b(?:Bun|Deno)\s*\.\s*(?:spawn|spawnSync|Command)\s*\(/);
138 expect(source).not.toMatch(/from\s+["'][^"']*(?:ChildProcess|CommandExecutor)[^"']*["']/);
139 }
140 });
141
142 export const successful = (event: TransportEvent) =>
143 event.status !== undefined && event.status >= 200 && event.status < 300;
144
145 export const boundary = (event: TransportEvent) => {
146 expect(successful(event)).toBe(true);
147 expect(event.machineId).toBeDefined();
148 return {
149 sequence: event.sequence,
150 method: event.method,
151 path: event.path,
152 status: event.status!,
153 machineId: event.machineId!,
154 };
155 };
156
157 export const firstReturnedUncordon = (events: readonly TransportEvent[], appName: string) =>
158 events.find(
159 (event) =>
160 event.stage === "forwarded" &&
161 successful(event) &&
162 event.method === "POST" &&
163 event.path.startsWith(`/v1/apps/${appName}/machines/`) &&
164 event.path.endsWith("/uncordon"),
165 );
166
167 export const matches = (
168 phase: SignalCase["phase"],
169 appName: string,
170 predecessorId: string,
171 events: readonly TransportEvent[],
172 event: TransportEvent,
173 ) => {
174 const base = `/v1/apps/${appName}/machines`;
175 if (phase === "create") return event.method === "POST" && event.path === base;
176 if (phase === "promotion")
177 return (
178 event.method === "POST" &&
179 event.path.startsWith(`${base}/`) &&
180 event.path.endsWith("/uncordon")
181 );
182 if (phase === "retirement")
183 return event.method === "DELETE" && event.path === `${base}/${predecessorId}`;
184 const returned = firstReturnedUncordon(events, appName);
185 return (
186 returned !== undefined &&
187 returned.machineId !== predecessorId &&
188 event.sequence > returned.sequence &&
189 event.method === "GET" &&
190 event.path === `${base}/${returned.machineId}`
191 );
192 };
193
194 export const assertBoundary = (witness: Witness) =>
195 Effect.sync(() => {
196 const { barrier, returnedUncordon } = witness;
197 expect(barrier.sequence).toBeGreaterThan(0);
198 expect(barrier.status).toBeGreaterThanOrEqual(200);
199 expect(barrier.status).toBeLessThan(300);
200 expect(barrier.machineId).toBe(
201 witness.phase === "retirement" ? witness.predecessor.id : witness.candidate.id,
202 );
203 if (witness.phase === "overlap") {
204 expect(returnedUncordon).toBeDefined();
205 expect(returnedUncordon!.method).toBe("POST");
206 expect(returnedUncordon!.path).toBe(
207 `/v1/apps/${witness.appName}/machines/${witness.candidate.id}/uncordon`,
208 );
209 expect(returnedUncordon!.machineId).toBe(witness.candidate.id);
210 expect(returnedUncordon!.status).toBeGreaterThanOrEqual(200);
211 expect(returnedUncordon!.status).toBeLessThan(300);
212 expect(returnedUncordon!.sequence).toBeGreaterThan(0);
213 expect(barrier.sequence).toBeGreaterThan(returnedUncordon!.sequence);
214 } else expect(returnedUncordon).toBeUndefined();
215 expect(
216 matches(
217 witness.phase,
218 witness.appName,
219 witness.predecessor.id,
220 returnedUncordon ? [{ ...returnedUncordon, stage: "forwarded" }] : [],
221 { ...barrier, stage: "held" },
222 ),
223 ).toBe(true);
224 });
225
226 export const assertInventory = (stack: ScratchStack, witness: Witness) =>
227 Effect.gen(function* () {
228 const live = yield* census(witness.appName);
229 expect(live.map((value) => value.id).sort()).toEqual(
230 (witness.phase === "retirement"
231 ? [witness.candidate.id]
232 : [witness.predecessor.id, witness.candidate.id]
233 ).sort(),
234 );
235 const candidate = yield* machine(witness.appName, witness.candidate.id);
236 expect(yield* identity(candidate, stack)).toEqual(witness.candidate);
237 expect(candidate.config?.env?.VERSION).toBe("two");
238 expect(candidate.cordoned).toBe(witness.phase === "create");
239 expect(witness.candidate.phase).toBe(
240 witness.phase === "create"
241 ? "candidate"
242 : witness.phase === "retirement"
243 ? "active"
244 : "promoting",
245 );
246 if (witness.phase !== "retirement") {
247 const predecessor = yield* machine(witness.appName, witness.predecessor.id);
248 expect(yield* identity(predecessor, stack)).toEqual(witness.predecessor);
249 expect(predecessor.cordoned).toBe(false);
250 expect(predecessor.state).toBe("started");
251 expect(predecessor.config?.env?.VERSION).toBe("one");
252 }
253 });
254
255 // Only presence and timestamps escape the SDK; nonce and transport headers never do.
256 export const observeLease = (appName: string, machineId: string) =>
257 machines.getMachineLease({ app_name: appName, machine_id: machineId }).pipe(
258 Retry.none,
259 Effect.map((value) => ({
260 present: !!value.data?.nonce,
261 expiresAt: value.data?.expires_at ?? 0,
262 })),
263 Effect.catchTag("NotFound", () => Effect.succeed({ present: false, expiresAt: 0 })),
264 Effect.timeout("10 seconds"),
265 Effect.mapError((error) => new Error(`Signal lease observation failed (${error._tag})`)),
266 Effect.flatMap((value) => nowSeconds.pipe(Effect.map((at) => ({ machineId, ...value, at })))),
267 );
268
269 export const observeLeases = (witness: Witness) =>
270 Effect.forEach(witness.leases, (held) => observeLease(witness.appName, held.machineId), {
271 concurrency: 2,
272 });
273
274 export const observeExpiry = (witness: Witness) =>
275 Effect.gen(function* () {
276 expect(witness.signal).toBe("SIGKILL");
277 expect(witness.leases.length).toBeGreaterThan(0);
278 const initial = yield* observeLeases(witness);
279 for (const [index, value] of initial.entries()) {
280 expect(value.present).toBe(true);
281 expect(value.expiresAt).toBeGreaterThan(value.at);
282 expect(value.expiresAt).toBeGreaterThanOrEqual(witness.leases[index]!.expiresAt);
283 expect(value.expiresAt).toBeLessThanOrEqual(witness.recordedAt + 130);
284 }
285 const samples: (typeof LeaseObservation.Type)[] = [];
286 const expired = yield* Effect.forEach(
287 initial,
288 (held) =>
289 Effect.gen(function* () {
290 const value = yield* observeLease(witness.appName, held.machineId);
291 yield* Effect.sync(() => samples.push(value));
292 if (value.present) expect(value.expiresAt).toBe(held.expiresAt);
293 else expect(value.at).toBeGreaterThanOrEqual(held.expiresAt);
294 return !value.present;
295 }),
296 { concurrency: 2 },
297 ).pipe(
298 Effect.repeat({
299 schedule: Schedule.spaced("3 seconds"),
300 times: 65,
301 until: (all) => all.every(Boolean),
302 }),
303 Effect.timeout("230 seconds"),
304 );
305 expect(expired.every(Boolean)).toBe(true);
306 for (const held of initial) {
307 const surviving = yield* machine(witness.appName, held.machineId);
308 expect(surviving.state).not.toBe("destroyed");
309 }
310 return { initial, samples };
311 });
312
313 export const assertReleased = (witness: Witness, finalized: typeof Finalized.Type) =>
314 Effect.gen(function* () {
315 expect(witness.signal).toBe("SIGINT");
316 expect(witness.leases.length).toBeGreaterThan(0);
317 expect(finalized.pid).toBe(witness.pid);
318 expect(finalized.case).toBe(witness.case);
319 expect(finalized.witnessRecordedAt).toBe(witness.recordedAt);
320 expect(finalized.interruptedOnly).toBe(true);
321 expect(finalized.at).toBeGreaterThanOrEqual(witness.recordedAt);
322 expect(finalized.leases.map((value) => value.machineId).sort()).toEqual(
323 witness.leases.map((value) => value.machineId).sort(),
324 );
325 for (const held of witness.leases) {
326 const observed = finalized.leases.find((value) => value.machineId === held.machineId)!;
327 expect(observed.present).toBe(false);
328 expect(observed.at).toBeGreaterThanOrEqual(witness.recordedAt);
329 expect(observed.at).toBeLessThan(held.expiresAt);
330 expect(finalized.at).toBeLessThan(held.expiresAt);
331 const release = finalized.releases.find(
332 (event) => event.machineId === held.machineId && event.status >= 200 && event.status < 300,
333 );
334 expect(release).toBeDefined();
335 expect(release!.method).toBe("DELETE");
336 expect(release!.path).toBe(`/v1/apps/${witness.appName}/machines/${held.machineId}/lease`);
337 expect(release!.sequence).toBeGreaterThan(witness.barrier.sequence);
338 // A deleted Machine cannot masquerade as a released lease.
339 const surviving = yield* machine(witness.appName, held.machineId);
340 expect(surviving.state).not.toBe("destroyed");
341 }
342 const current = yield* observeLeases(witness);
343 expect(current.every((value) => !value.present)).toBe(true);
344 return { finalized, current };
345 });
346
347 export const assertConverged = (stack: ScratchStack, witness: Witness, ids: string[]) =>
348 Effect.gen(function* () {
349 expect(ids).toEqual([witness.candidate.id]);
350 const live = yield* census(witness.appName);
351 expect(live.map((value) => value.id)).toEqual(ids);
352 const current = yield* machine(witness.appName, witness.candidate.id).pipe(
353 Effect.repeat({
354 schedule: Schedule.spaced("2 seconds"),
355 times: 20,
356 until: (value) =>
357 value.state === "started" &&
358 value.cordoned === false &&
359 value.checks?.some((check) => check.name === "ready" && check.status === "passing") ===
360 true,
361 }),
362 Effect.timeout("60 seconds"),
363 );
364 expect(yield* identity(current, stack)).toEqual({
365 ...witness.candidate,
366 phase: "active",
367 });
368 expect(current.state).toBe("started");
369 expect(current.cordoned).toBe(false);
370 expect(current.config?.env?.VERSION).toBe("two");
371 const ready = current.checks?.filter((check) => check.name === "ready") ?? [];
372 expect(ready.length).toBeGreaterThan(0);
373 expect(ready.every((check) => check.status === "passing")).toBe(true);
374 expect(current.instance_id).toBeDefined();
375 expect(current.config?.metadata?.["alchemy.checked-instance"]).toBe(current.instance_id);
376 expect(current.config?.metadata?.["alchemy.predecessors"]).toBe(witness.predecessor.id);
377 expect(
378 current.config?.metadata?.["alchemy.image"]?.endsWith(`@${witness.candidate.digest}`),
379 ).toBe(true);
380 const row = yield* persistedRow(stack, witness.row.fqn);
381 expect(row.instanceId).toBe(witness.row.instanceId);
382 expect(["created", "updated"]).toContain(row.status);
383 });