The best way to control Claude Code, Codex and more from any computer or phone

main
ts 536 lines 20.7 KB
Raw
1 import { expect } from "@alchemy.run/test";
2 import type { ScratchStack } from "@alchemy.run/test/Alchemy";
3 import { scratchStack } from "@alchemy.run/test/Core";
4 import type * as machines from "@distilled.cloud/fly-io/machines";
5 import * as Effect from "effect/Effect";
6 import * as HttpClient from "effect/http/HttpClient";
7 import * as Layer from "effect/Layer";
8 import { throughProxy, transportProxy, type TransportEvent } from "./transport.ts";
9
10 const metadataKeys = [
11 "alchemy.stack",
12 "alchemy.stage",
13 "alchemy.id",
14 "alchemy.type",
15 "alchemy.replica",
16 "alchemy.instance",
17 "alchemy.fqn",
18 "alchemy.base-name",
19 "alchemy.generation",
20 "alchemy.workload",
21 "alchemy.sequence",
22 "alchemy.count",
23 "alchemy.image",
24 "alchemy.phase",
25 "alchemy.deployment-protocol",
26 "alchemy.readiness-roles",
27 "alchemy.readiness-role",
28 "alchemy.predecessors",
29 "alchemy.idle-policy-restored",
30 "alchemy.checked-instance",
31 "alchemy.min-secrets-version",
32 ];
33
34 const record = (value: unknown): Record<string, unknown> =>
35 typeof value === "object" && value !== null && !Array.isArray(value)
36 ? (value as Record<string, unknown>)
37 : {};
38 const string = (value: unknown) => (typeof value === "string" ? value : undefined);
39 const metadataOf = (value: unknown) => {
40 const metadata = record(value);
41 return Object.fromEntries(
42 metadataKeys.flatMap((key) =>
43 typeof metadata[key] === "string" ? [[key, metadata[key] as string]] : [],
44 ),
45 );
46 };
47 const servicesOf = (value: unknown) =>
48 Array.isArray(value)
49 ? value.map((item) => {
50 const service = record(item);
51 return {
52 protocol: string(service.protocol),
53 port: typeof service.internal_port === "number" ? service.internal_port : undefined,
54 autostop:
55 service.autostop === true
56 ? "stop"
57 : service.autostop === "stop" || service.autostop === "suspend"
58 ? service.autostop
59 : "off",
60 autostart: typeof service.autostart === "boolean" ? service.autostart : undefined,
61 floor:
62 typeof service.min_machines_running === "number"
63 ? service.min_machines_running
64 : undefined,
65 ...(Array.isArray(service.checks)
66 ? {
67 checks: service.checks.map((value) => {
68 const check = record(value);
69 return {
70 type: string(check.type),
71 port: typeof check.port === "number" ? check.port : undefined,
72 interval: string(check.interval),
73 timeout: string(check.timeout),
74 gracePeriod: string(check.grace_period),
75 method: string(check.method),
76 protocol: string(check.protocol),
77 };
78 }),
79 }
80 : {}),
81 };
82 })
83 : [];
84
85 export interface ReadinessEvent extends TransportEvent {
86 metadata?: Record<string, string>;
87 metadataOnly?: boolean;
88 skipLaunch?: boolean;
89 services?: ReturnType<typeof servicesOf>;
90 serviceConfig?: string;
91 }
92
93 /** Only these two excluded suites install this observer; bodies and headers are never retained. */
94 export const readinessProxy = () =>
95 transportProxy().pipe(
96 Effect.map((proxy) => ({ ...proxy, readiness: [] as ReadinessEvent[] })),
97 Effect.tap((proxy) =>
98 Effect.addFinalizer(() =>
99 Effect.forEach(
100 Array.from({ length: Math.ceil(proxy.readiness.length / 50) }, (_, index) => index),
101 (index) =>
102 Effect.logInfo("Idle/cadence readiness journal", {
103 offset: index * 50,
104 events: proxy.readiness.slice(index * 50, (index + 1) * 50),
105 }),
106 { discard: true },
107 ),
108 ),
109 ),
110 );
111
112 type ReadinessProxy = Effect.Success<ReturnType<typeof readinessProxy>>;
113
114 export const readinessActor = (
115 parent: ScratchStack,
116 title: string,
117 file: string,
118 proxy: ReadinessProxy,
119 beforeRequest?: (event: ReadinessEvent) => Effect.Effect<void>,
120 ) =>
121 Effect.sync(() => {
122 const observer = Layer.effect(
123 HttpClient.HttpClient,
124 Effect.gen(function* () {
125 const client = yield* HttpClient.HttpClient;
126 return HttpClient.transform(client, (response, request) =>
127 Effect.gen(function* () {
128 const event = yield* Effect.sync(() => {
129 const path = new URL(request.url).pathname;
130 if (!/\/apps\/[^/]+\/machines(?:\/|$)/.test(path)) return undefined;
131 const event: ReadinessEvent = {
132 sequence: proxy.readiness.length,
133 stage: "request",
134 method: request.method,
135 path,
136 machineId: path.match(/\/machines\/([^/]+)/)?.[1],
137 };
138 if (request.body._tag === "Uint8Array") {
139 try {
140 const body = record(JSON.parse(new TextDecoder().decode(request.body.body)));
141 const config = record(body.config);
142 const metadata = record(body.metadata ?? config.metadata);
143 event.metadata = metadataOf(metadata);
144 event.phase = event.metadata["alchemy.phase"];
145 event.skipLaunch =
146 typeof body.skip_launch === "boolean" ? body.skip_launch : undefined;
147 event.metadataOnly =
148 Object.keys(body).length === 1 &&
149 "metadata" in body &&
150 Object.keys(metadata).every(
151 (key) => metadataKeys.includes(key) && typeof metadata[key] === "string",
152 );
153 } catch {
154 // An unreadable body cannot qualify for the bookkeeping exemption.
155 }
156 }
157 proxy.readiness.push(event);
158 return event;
159 });
160 if (event && beforeRequest) yield* beforeRequest(event);
161 const result = yield* response;
162 if (event) {
163 const value =
164 result.status >= 200 &&
165 result.status < 300 &&
166 /\/machines(?:\/[^/]+)?$/.test(event.path)
167 ? yield* result.json.pipe(Effect.catch(() => Effect.succeed(undefined)))
168 : undefined;
169 yield* Effect.sync(() => {
170 const machine = record(value);
171 const config = record(machine.config);
172 const observed: ReadinessEvent = {
173 ...event,
174 // Receipt here is after the unmodified proxy forwarded the response.
175 stage: "forwarded",
176 status: result.status,
177 };
178 if (typeof machine.id === "string") {
179 observed.machineId = machine.id;
180 observed.instanceId = string(machine.instance_id);
181 observed.state = string(machine.state);
182 observed.cordoned =
183 typeof machine.cordoned === "boolean" ? machine.cordoned : undefined;
184 observed.digest = string(record(machine.image_ref).digest);
185 observed.metadata = metadataOf(config.metadata);
186 observed.phase = observed.metadata["alchemy.phase"];
187 observed.services = servicesOf(config.services);
188 observed.serviceConfig = JSON.stringify(observed.services);
189 observed.checks = Array.isArray(machine.checks)
190 ? machine.checks.map((item) => {
191 const check = record(item);
192 return {
193 name: string(check.name),
194 status: string(check.status),
195 };
196 })
197 : [];
198 }
199 proxy.readiness.push(observed);
200 });
201 }
202 return result;
203 }),
204 );
205 }),
206 ).pipe(Layer.provideMerge(throughProxy(() => proxy.url)));
207 const actor = scratchStack({ providers: observer, stage: parent.stage }, title, file);
208 expect(actor.name).toBe(parent.name);
209 expect(actor.stage).toBe(parent.stage);
210 expect(actor.state).not.toBe(parent.state);
211 return actor;
212 });
213
214 export const retires = (event: TransportEvent, ids: readonly string[]) =>
215 event.stage === "request" &&
216 ids.includes(event.machineId!) &&
217 (event.path.endsWith("/cordon") ||
218 event.path.endsWith("/stop") ||
219 event.path.endsWith("/suspend") ||
220 (event.path.endsWith("/metadata") && event.phase === "retiring") ||
221 (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path)));
222
223 const identity = (metadata: Record<string, string> | undefined) =>
224 Object.fromEntries(
225 Object.entries(metadata ?? {}).filter(
226 ([key]) => key !== "alchemy.phase" && key !== "alchemy.checked-instance",
227 ),
228 );
229
230 export const readinessChecksPassing = (
231 checks: ReadinessEvent["checks"],
232 expected: readonly string[],
233 ) => {
234 if (!checks || expected.length === 0 || new Set(expected).size !== expected.length) return false;
235 const allowed = new Set([
236 ...expected,
237 ...expected
238 .filter((name) => name.startsWith("servicecheck-"))
239 .map((name) => `bg_deployments_compat-${name.slice("servicecheck-".length)}`),
240 ]);
241 const reported = new Set(checks.map((check) => check.name));
242 return (
243 reported.size === checks.length &&
244 expected.every((name) => reported.has(name)) &&
245 checks.every(
246 (check) => check.name !== undefined && allowed.has(check.name) && check.status === "passing",
247 )
248 );
249 };
250
251 const sameMetadata = (left: ReadinessEvent, right: ReadinessEvent) =>
252 left.method === "PATCH" &&
253 right.method === "PATCH" &&
254 left.path.endsWith("/metadata") &&
255 left.path === right.path &&
256 left.metadataOnly === true &&
257 right.metadataOnly === true &&
258 left.metadata !== undefined &&
259 right.metadata !== undefined &&
260 JSON.stringify(Object.entries(left.metadata).sort()) ===
261 JSON.stringify(Object.entries(right.metadata).sort());
262
263 /** Fresh GET receipt proves observation order, not the remote check sample's timestamp. */
264 export const assertReadinessCommit = (
265 events: readonly ReadinessEvent[],
266 priorIds: readonly string[],
267 candidates: machines.Machine[],
268 runningSlots: readonly number[],
269 checkNames: readonly string[],
270 allowIdle: boolean,
271 ) => {
272 const retirement =
273 priorIds.length > 0 ? events.findIndex((event) => retires(event, priorIds)) : events.length;
274 expect(retirement).toBeGreaterThan(0);
275 const ordered = [...candidates].sort(
276 (a, b) =>
277 Number(a.config?.metadata?.["alchemy.replica"]) -
278 Number(b.config?.metadata?.["alchemy.replica"]),
279 );
280 const ids = ordered.map((machine) => machine.id!);
281 const creates = events
282 .filter(
283 (event) =>
284 event.stage === "forwarded" &&
285 event.method === "POST" &&
286 event.path.endsWith("/machines") &&
287 ids.includes(event.machineId!),
288 )
289 .map((event) => event.sequence);
290 const mutations = events.filter(
291 (event) =>
292 event.stage === "request" &&
293 event.method !== "GET" &&
294 !event.path.endsWith("/lease") &&
295 (ids.includes(event.machineId!) || creates.includes(event.sequence)),
296 );
297 // Phase stamps go through Fly's `PATCH /machines/{id}/metadata` (formerly PUT).
298 const isCommit = (event: ReadinessEvent) =>
299 event.method === "PATCH" &&
300 event.path.endsWith("/metadata") &&
301 event.phase === "active" &&
302 event.metadataOnly === true;
303 const isValidating = (event: ReadinessEvent) =>
304 event.method === "PATCH" &&
305 event.path.endsWith("/metadata") &&
306 event.phase === "validating" &&
307 event.metadataOnly === true;
308 const successful = (event: ReadinessEvent) => event.status! >= 200 && event.status! < 300;
309 const operations: {
310 request: ReadinessEvent;
311 machineId: string;
312 completed: number;
313 }[] = [];
314 for (const request of mutations) {
315 const receipts = events.filter(
316 (event) =>
317 event.sequence === request.sequence &&
318 event.stage === "forwarded" &&
319 event.method === request.method &&
320 event.path === request.path,
321 );
322 expect(receipts).toHaveLength(1);
323 const receipt = receipts[0]!;
324 const completed = events.indexOf(receipt);
325 const requested = events.indexOf(request);
326 expect(completed).toBeGreaterThan(requested);
327 expect(completed).toBeLessThan(retirement);
328 const machineId = request.machineId ?? receipt.machineId!;
329 if (request.path.endsWith("/start") && receipt.status === 412) {
330 const candidate = ordered.find((machine) => machine.id === machineId)!;
331 const commit = events.findIndex((event) => isCommit(event) && event.machineId === machineId);
332 expect(
333 events.some(
334 (event, index) =>
335 index > completed &&
336 index < commit &&
337 event.stage === "forwarded" &&
338 event.method === "GET" &&
339 successful(event) &&
340 event.machineId === machineId &&
341 event.state === "started" &&
342 event.instanceId === candidate.instance_id,
343 ),
344 ).toBe(true);
345 continue;
346 }
347 const previous = operations.findLast((operation) => operation.machineId === machineId);
348 // Only settled, retryable metadata failures may join an equivalent retry.
349 if (
350 previous &&
351 !successful(events[previous.completed]!) &&
352 sameMetadata(previous.request, request)
353 ) {
354 expect([429, 500, 502, 503, 504]).toContain(events[previous.completed]!.status);
355 expect(previous.completed).toBeLessThan(requested);
356 previous.completed = completed;
357 } else {
358 operations.push({ request, machineId, completed });
359 }
360 }
361 for (const operation of operations) {
362 expect(successful(events[operation.completed]!)).toBe(true);
363 }
364 const commits = operations.filter((operation) => isCommit(operation.request));
365 expect(commits).toHaveLength(ordered.length);
366 expect(commits.map((operation) => operation.machineId).sort()).toEqual([...ids].sort());
367 expect(commits.at(-1)?.machineId).toBe(ordered[0]!.id);
368 const firstCommit = events.indexOf(commits[0]!.request);
369 expect(firstCommit).toBeGreaterThan(0);
370 const invalidating = operations.filter((operation) => !isCommit(operation.request));
371 expect(invalidating.length).toBeGreaterThan(0);
372 const reset = Math.max(...invalidating.map((operation) => operation.completed));
373 expect(reset).toBeLessThan(firstCommit);
374 const zeroCommit = events.indexOf(commits.at(-1)!.request);
375 for (const operation of commits.slice(0, -1)) {
376 expect(operation.completed).toBeLessThan(zeroCommit);
377 }
378
379 for (const [slot, machine] of ordered.entries()) {
380 const id = machine.id!;
381 const run = runningSlots.includes(slot);
382 const commit = commits.find((operation) => operation.machineId === id)!.request;
383 const finalMetadata = metadataOf(machine.config?.metadata);
384 const finalServices = servicesOf(machine.config?.services);
385 if (allowIdle) {
386 expect(finalServices.length).toBeGreaterThan(0);
387 expect(finalServices.every((service) => service.autostop !== "off")).toBe(true);
388 }
389 expect(finalMetadata["alchemy.replica"]).toBe(String(slot));
390 expect(finalMetadata["alchemy.readiness-role"]).toBe(run ? "run" : "idle");
391 expect(finalMetadata["alchemy.readiness-roles"]).toBe(
392 ordered.map((_, index) => (runningSlots.includes(index) ? "run" : "idle")).join(","),
393 );
394 expect(finalMetadata["alchemy.phase"]).toBe("active");
395 expect(finalMetadata["alchemy.idle-policy-restored"]).toBe("true");
396 expect(commit.metadata).toEqual(finalMetadata);
397 expect(machine.cordoned).toBe(false);
398 for (const key of [
399 "alchemy.stack",
400 "alchemy.stage",
401 "alchemy.id",
402 "alchemy.type",
403 "alchemy.instance",
404 "alchemy.fqn",
405 "alchemy.generation",
406 "alchemy.workload",
407 "alchemy.image",
408 ]) {
409 expect(finalMetadata[key]).toBeDefined();
410 }
411 const observations = events.flatMap((event, index) => {
412 if (
413 index >= firstCommit ||
414 event.stage !== "forwarded" ||
415 event.method !== "GET" ||
416 !event.path.endsWith(`/machines/${id}`) ||
417 event.machineId !== id ||
418 event.status !== 200
419 )
420 return [];
421 const requested = events.findIndex(
422 (request) => request.stage === "request" && request.sequence === event.sequence,
423 );
424 return requested >= 0 && requested < index ? [{ event, index, requested }] : [];
425 });
426 const pending = observations.findLast(({ requested }) => requested > reset);
427 expect(pending).toBeDefined();
428 if (!pending) continue;
429 expect(pending.event.phase).toBe("validating");
430 expect(pending.event.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
431 expect(pending.event.instanceId).toBe(machine.instance_id);
432 let proof = pending;
433 if (run && pending.event.state !== "started") {
434 expect(allowIdle).toBe(true);
435 expect(["stopped", "suspended"]).toContain(pending.event.state);
436 const targetReset = Math.max(
437 -1,
438 ...invalidating
439 .filter((operation) => operation.machineId === id && !isValidating(operation.request))
440 .map((operation) => operation.completed),
441 );
442 const restored = observations.findLast(
443 ({ event, requested, index }) =>
444 requested > targetReset &&
445 index < pending.index &&
446 ["promoting", "validating"].includes(event.phase ?? "") &&
447 event.metadata?.["alchemy.idle-policy-restored"] === "true" &&
448 event.instanceId === pending.event.instanceId &&
449 event.state === "started" &&
450 readinessChecksPassing(event.checks, checkNames),
451 );
452 expect(restored).toBeDefined();
453 if (!restored) continue;
454 proof = restored;
455 } else if (run) {
456 expect(readinessChecksPassing(proof.event.checks, checkNames)).toBe(true);
457 } else {
458 expect(["created", "stopped", "suspended"]).toContain(pending.event.state);
459 }
460 expect(proof.event.instanceId).toBeDefined();
461 expect(proof.event.digest).toBe(machine.image_ref?.digest);
462 expect(proof.event.services).toEqual(finalServices);
463 expect(proof.event.cordoned).toBe(false);
464 expect(identity(proof.event.metadata)).toEqual(identity(finalMetadata));
465 expect(commit.metadata).toEqual({
466 ...pending.event.metadata,
467 "alchemy.phase": "active",
468 ...(run ? { "alchemy.checked-instance": proof.event.instanceId } : {}),
469 });
470 if (run) {
471 expect(commit.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId);
472 expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain(
473 machine.state,
474 );
475 } else {
476 expect(["created", "stopped", "suspended"]).toContain(machine.state);
477 for (const created of events.filter(
478 (event) =>
479 event.stage === "forwarded" &&
480 event.method === "POST" &&
481 event.path.endsWith("/machines") &&
482 event.machineId === id,
483 )) {
484 expect(created.skipLaunch).toBe(true);
485 }
486 expect(
487 events.some(
488 (event) =>
489 event.machineId === id &&
490 ((event.stage === "request" && event.path.endsWith("/start")) ||
491 (event.stage === "forwarded" && event.state === "started")),
492 ),
493 ).toBe(false);
494 }
495 // Idle proof survives only the unchanged pending stamp and final commit.
496 for (const operation of invalidating.filter(
497 (operation) => operation.machineId === id && operation.completed > proof.index,
498 )) {
499 expect(isValidating(operation.request)).toBe(true);
500 expect(events.indexOf(operation.request)).toBeGreaterThan(proof.index);
501 expect(operation.request.metadata).toEqual({
502 ...proof.event.metadata,
503 "alchemy.phase": "validating",
504 });
505 }
506 for (const later of events
507 .slice(proof.index + 1)
508 .filter(
509 (event) =>
510 event.stage === "forwarded" &&
511 event.machineId === id &&
512 event.status! >= 200 &&
513 event.status! < 300 &&
514 event.path.endsWith(`/machines/${id}`),
515 )) {
516 expect(later.instanceId).toBe(proof.event.instanceId);
517 expect(later.digest).toBe(proof.event.digest);
518 expect(later.services).toEqual(finalServices);
519 expect(identity(later.metadata)).toEqual(identity(finalMetadata));
520 expect(later.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
521 expect(["promoting", "validating", "active"]).toContain(later.phase);
522 expect(later.cordoned).toBe(false);
523 if (run) {
524 expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain(
525 later.state,
526 );
527 if (later.state === "started" && events.indexOf(later) < firstCommit) {
528 expect(readinessChecksPassing(later.checks, checkNames)).toBe(true);
529 }
530 if (later.phase === "active") {
531 expect(later.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId);
532 }
533 }
534 }
535 }
536 };