@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 1,480 lines 51.2 KB
Raw
1 import type {
2 FlyMachineConfig,
3 FlyMachineMount,
4 FlyMachineService,
5 FlyMachineServiceCheck,
6 FlyStopConfig,
7 ImageRef as FlyImageRef,
8 Machine as FlyMachine,
9 Volume as FlyVolume,
10 } from "@distilled.cloud/fly-io/machines";
11 import * as machines from "@distilled.cloud/fly-io/machines";
12 import * as Retry from "@distilled.cloud/fly-io/Retry";
13 import { deepEqual } from "alchemy/Diff";
14 import * as Data from "effect/Data";
15 import * as Effect from "effect/Effect";
16 import * as Result from "effect/Result";
17 import * as Schedule from "effect/Schedule";
18 import { listOwnedApps } from "./App.ts";
19 import { reconcileBlueGreen, setRouting } from "./bluegreen.ts";
20 import { validateDeployment, type DeploymentPolicy, type MachineCheck } from "./Deployment.ts";
21 import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts";
22 import { usingMachineLeases, type MachineLeases } from "./leases.ts";
23 import type {
24 MachineGuest,
25 MachineImageRef,
26 MachineService,
27 MachineServiceCheck,
28 } from "./Machine.ts";
29 import {
30 alchemyMetadataKeys,
31 createMachineMetadata,
32 isAlchemyOwnedMetadata,
33 sanitizeFlyAppName,
34 type FlyAlchemyType,
35 } from "./Metadata.ts";
36 import type { DiskSpec, MountedDisk } from "./MountVolume.ts";
37 import { regionOfReplica } from "./Region.ts";
38 import { deleteVolume, ensureVolumeGroup, getVolumeById, volumeGroupName } from "./Volume.ts";
39
40 const WAIT_TIMEOUT_SECONDS = 8;
41 const waitBackoff = Schedule.exponential("500 millis");
42 const SERVICE_CHECK_NAME_PREFIX = "servicecheck-";
43
44 export class ReplicaNotCreated extends Data.TaggedError("Fly.ReplicaNotCreated")<{
45 resourceName: string;
46 appName: string;
47 }> {}
48
49 export class ReplicaChecksNotPassing extends Data.TaggedError("Fly.ReplicaChecksNotPassing")<{
50 appName: string;
51 machineId: string;
52 checks: ReadonlyArray<{
53 name: string | undefined;
54 status: string | undefined;
55 output: string | undefined;
56 }>;
57 }> {
58 get message() {
59 const checks = this.checks.map(
60 (check) =>
61 `${check.name ?? "unnamed"}: ${check.status ?? "unknown"}${check.output ? ` (${check.output})` : ""}`,
62 );
63 return `Service checks did not pass for ${this.appName}/${this.machineId}: ${checks.join("; ") || "no service check results"}`;
64 }
65 }
66
67 export interface Replica {
68 machineId: string;
69 name: string;
70 baseName?: string;
71 region: string;
72 state: string;
73 instanceId: string | undefined;
74 privateIp: string | undefined;
75 imageRef: MachineImageRef | undefined;
76 guest: MachineGuest | undefined;
77 mounts: MountedDisk[];
78 }
79
80 export interface ReplicaSet {
81 rolloutPending?: boolean;
82 appName: string;
83 machineId: string;
84 machineIds: string[];
85 name: string;
86 baseName?: string;
87 /** Region of replica 0. */
88 region: string;
89 /** Regions the replicas run in, in replica order. */
90 regions: string[];
91 state: string;
92 instanceId: string | undefined;
93 privateIp: string | undefined;
94 imageRef: MachineImageRef | undefined;
95 guest: MachineGuest | undefined;
96 /** Observed proxy services published by replica 0. */
97 services: FlyMachineService[] | undefined;
98 count: number;
99 mounts: MountedDisk[];
100 replicas: Replica[];
101 }
102
103 const compactRecord = (
104 record: Record<string, string | undefined> | null | undefined,
105 ): Record<string, string> =>
106 Object.fromEntries(
107 Object.entries(record ?? {}).flatMap(([key, value]) =>
108 value === undefined ? [] : [[key, value]],
109 ),
110 );
111
112 export const gone = (machine: FlyMachine | undefined) =>
113 machine === undefined || machine.state === "destroyed";
114
115 export const getMachineById = (appName: string, machineId: string) =>
116 machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
117 Effect.map((machine) => (gone(machine) ? undefined : machine)),
118 Effect.catchTag("NotFound", () => Effect.undefined),
119 );
120
121 /**
122 * Reads a Machine we hold a lease on. Fly's Machines API is not
123 * read-after-write consistent: a freshly created Machine can accept a lease
124 * (201) and then answer `GET /machines/{id}` with 404 for several seconds.
125 * Holding the lease blocks other writers, so a 404 here is lag rather than
126 * deletion; retry it (bounded) before surfacing `NotFound`.
127 */
128 const getLeasedMachine = (appName: string, machineId: string) =>
129 machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
130 Effect.retry({
131 times: 10,
132 schedule: Schedule.spaced("1 second"),
133 while: (error) => error._tag === "NotFound",
134 }),
135 );
136
137 const getLeasedMachineById = (appName: string, machineId: string) =>
138 getLeasedMachine(appName, machineId).pipe(
139 Effect.map((machine) => (gone(machine) ? undefined : machine)),
140 Effect.catchTag("NotFound", () => Effect.undefined),
141 );
142
143 export const listMachinesByApp = (appName: string) =>
144 machines.listMachines({ app_name: appName }).pipe(
145 // A just-created App can answer 500 "failed to get app: sql: no rows in
146 // result set" until the Machines API sees it; safe to replay a read.
147 Effect.retry({
148 while: (error) => error._tag === "InternalServerError",
149 schedule: Schedule.spaced("2 seconds"),
150 times: 8,
151 }),
152 Effect.map((machines) => machines.filter((machine) => !gone(machine))),
153 Effect.catchTag("NotFound", () => Effect.succeed([])),
154 );
155
156 export const resolveCount = (count: number | undefined) => Math.max(1, Math.floor(count ?? 1));
157
158 export const replicaMachineName = (base: string, index: number, count: number) => {
159 if (count <= 1 && index === 0) return base;
160 const suffix = `-${index}`;
161 const room = 30 - suffix.length;
162 const clipped = base.slice(0, Math.max(1, room)).replace(/-+$/g, "");
163 return sanitizeFlyAppName(`${clipped}${suffix}`);
164 };
165
166 export const replicaIndexOf = (machine: FlyMachine): number => {
167 const raw = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.replica];
168 const parsed = raw === undefined ? 0 : Number(raw);
169 return Number.isFinite(parsed) ? parsed : 0;
170 };
171
172 export const alchemyIdOf = (machine: FlyMachine): string | undefined => {
173 const id = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.id];
174 return id !== undefined && id.length > 0 ? id : undefined;
175 };
176
177 export const isOwnedType = (machine: FlyMachine, type: FlyAlchemyType) => {
178 const metadata = compactRecord(machine.config?.metadata);
179 return isAlchemyOwnedMetadata(metadata) && metadata[alchemyMetadataKeys.type] === type;
180 };
181
182 export const toImageRef = (ref: FlyImageRef | undefined): MachineImageRef | undefined => {
183 if (ref === undefined) return undefined;
184 const imageRef: MachineImageRef = {
185 registry: ref.registry,
186 repository: ref.repository,
187 tag: ref.tag,
188 digest: ref.digest,
189 };
190 return imageRef.registry === undefined &&
191 imageRef.repository === undefined &&
192 imageRef.tag === undefined &&
193 imageRef.digest === undefined
194 ? undefined
195 : imageRef;
196 };
197
198 export const toGuestAttrs = (
199 guest:
200 | { cpu_kind?: string; cpus?: number; memory_mb?: number; gpu_kind?: string; gpus?: number }
201 | undefined,
202 ): MachineGuest | undefined => {
203 if (guest === undefined) return undefined;
204 return {
205 cpuKind: guest.cpu_kind,
206 cpus: guest.cpus,
207 memoryMb: guest.memory_mb,
208 gpuKind: guest.gpu_kind,
209 gpus: guest.gpus,
210 };
211 };
212
213 export const toFlyServiceCheck = (check: MachineServiceCheck): FlyMachineServiceCheck => ({
214 type: check.type,
215 port: check.port,
216 interval: check.interval,
217 timeout: check.timeout,
218 grace_period: check.gracePeriod,
219 method: check.method,
220 path: check.path,
221 protocol: check.protocol,
222 headers: check.headers?.map((header) => ({ name: header.name, values: header.values })),
223 tls_server_name: check.tlsServerName,
224 tls_skip_verify: check.tlsSkipVerify,
225 });
226
227 export const toFlyService = (service: MachineService): FlyMachineService => ({
228 protocol: service.protocol,
229 internal_port: service.internalPort,
230 autostart: service.autostart,
231 autostop:
232 typeof service.autostop === "boolean" ? (service.autostop ? "stop" : "off") : service.autostop,
233 min_machines_running: service.minMachinesRunning,
234 ports: service.ports?.map((port) => ({
235 port: port.port,
236 handlers: port.handlers,
237 force_https: port.forceHttps,
238 start_port: port.startPort,
239 end_port: port.endPort,
240 })),
241 checks: service.checks?.map((check) => ({
242 ...toFlyServiceCheck(check),
243 // Fly clamps service-check intervals; named Machine checks retain longer intervals.
244 interval:
245 (durationNanoseconds(check.interval) ?? 0n) > 60_000_000_000n ? "60s" : check.interval,
246 })),
247 });
248
249 export const autostopMode = (value: string | boolean | undefined) =>
250 value === true ? "stop" : value === false || value === undefined ? "off" : value;
251
252 const normalizedCheckDuration = (value: string | undefined) => {
253 const nanos = durationNanoseconds(value);
254 return nanos === undefined ? value : `${nanos}ns`;
255 };
256
257 const normalizedCheck = (check: machines.FlyMachineCheck | FlyMachineServiceCheck | undefined) =>
258 check === undefined
259 ? undefined
260 : {
261 ...check,
262 grace_period: normalizedCheckDuration(check.grace_period),
263 interval: normalizedCheckDuration(check.interval),
264 timeout: normalizedCheckDuration(check.timeout),
265 };
266
267 export const sameChecks = (
268 observed: FlyMachineConfig["checks"],
269 desired: FlyMachineConfig["checks"],
270 ) => {
271 const normalize = (checks: FlyMachineConfig["checks"]) =>
272 Object.fromEntries(
273 Object.entries(checks ?? {}).map(([name, check]) => [name, normalizedCheck(check)]),
274 );
275 return deepEqual(normalize(observed), normalize(desired), { stripNullish: true });
276 };
277
278 export const normalizedServices = (services: FlyMachineService[] | undefined) =>
279 (services ?? []).map((service) => ({
280 ...service,
281 autostop: autostopMode(service.autostop),
282 checks: service.checks?.map(normalizedCheck),
283 }));
284
285 export const sameServices = (
286 observed: FlyMachineService[] | undefined,
287 desired: FlyMachineService[] | undefined,
288 ) => deepEqual(normalizedServices(observed), normalizedServices(desired), { stripNullish: true });
289
290 export const hasPublishedService = (services: FlyMachineService[] | undefined) =>
291 (services ?? []).some((service) =>
292 (service.ports ?? []).some((port) => port.port !== undefined || port.start_port !== undefined),
293 );
294
295 export const waitStarted = (appName: string, machineId: string) =>
296 machines
297 .waitMachine({
298 app_name: appName,
299 machine_id: machineId,
300 state: "started",
301 timeout: WAIT_TIMEOUT_SECONDS,
302 })
303 .pipe(
304 Retry.none,
305 Effect.retry({
306 times: 6,
307 schedule: waitBackoff,
308 while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
309 }),
310 Effect.timeout("50 seconds"),
311 );
312
313 export const configuredCheckNames = (config: FlyMachineConfig | undefined) => [
314 ...Object.keys(config?.checks ?? {}),
315 ...(config?.services ?? []).flatMap((service) =>
316 (service.checks ?? []).map(
317 (check, checkIndex) =>
318 `${SERVICE_CHECK_NAME_PREFIX}${String(checkIndex).padStart(2, "0")}-${check.type ?? "tcp"}-${service.internal_port}`,
319 ),
320 ),
321 ];
322
323 export const checksPassing = (machine: FlyMachine, config: FlyMachineConfig | undefined) => {
324 const expected = configuredCheckNames(config);
325 const checks = machine.checks ?? [];
326 // Fly may also report compatibility mirrors; they do not replace service reports.
327 return (
328 machine.state === "started" &&
329 new Set(expected).size === expected.length &&
330 checks.every(
331 (check) =>
332 check.name !== undefined &&
333 check.status === "passing" &&
334 (expected.includes(check.name) ||
335 expected.some(
336 (name) =>
337 name.startsWith(SERVICE_CHECK_NAME_PREFIX) &&
338 check.name === name.replace(SERVICE_CHECK_NAME_PREFIX, "bg_deployments_compat-"),
339 )),
340 ) &&
341 new Set(checks.map((check) => check.name)).size === checks.length &&
342 expected.every((name) => {
343 const reports = checks.filter((check) => check.name === name);
344 return reports.length === 1 && reports[0]?.status === "passing";
345 })
346 );
347 };
348
349 const TRANSIENT_GET_TAGS = [
350 "TooManyRequests",
351 "InternalServerError",
352 "BadGateway",
353 "ServiceUnavailable",
354 "GatewayTimeout",
355 ] as const;
356
357 /**
358 * After the Machine is started, wait until Fly reports every service
359 * check as passing. No configured checks is a no-op. Empty live
360 * `servicecheck-*` results keep polling — they are not success.
361 * `warning` / `unknown` during grace keep polling.
362 */
363 export const waitHealthy = Effect.fn(function* (
364 appName: string,
365 machine: FlyMachine,
366 healthTimeoutMs = 60_000,
367 config: FlyMachineConfig | undefined = machine.config,
368 ) {
369 const machineId = machine.id;
370 const named = Object.keys(config?.checks ?? {});
371 const expected = (config?.services ?? []).flatMap((service) => service.checks ?? []).length;
372 if (machineId === undefined || (expected === 0 && named.length === 0)) return machine;
373
374 let observed = machine;
375 const notPassing = () =>
376 new ReplicaChecksNotPassing({
377 appName,
378 machineId,
379 checks: (observed.checks ?? []).map((check) => ({
380 name: check.name,
381 status: check.status,
382 output: undefined,
383 })),
384 });
385 const passing = yield* getMachineById(appName, machineId).pipe(
386 Retry.none,
387 Effect.map((current) => {
388 if (current === undefined) return false;
389 observed = current;
390 return (
391 current.instance_id !== undefined &&
392 current.instance_id === machine.instance_id &&
393 checksPassing(current, config)
394 );
395 }),
396 Effect.catchTag(TRANSIENT_GET_TAGS, () => Effect.succeed(false)),
397 Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
398 Effect.repeat({
399 schedule: Schedule.spaced(healthTimeoutMs / 10),
400 until: (passing) => passing,
401 times: 10,
402 }),
403 Effect.timeoutOrElse({ duration: healthTimeoutMs, orElse: () => Effect.fail(notPassing()) }),
404 );
405 if (!passing) return yield* notPassing();
406 return observed;
407 });
408
409 export const waitDestroyed = (appName: string, machineId: string) =>
410 machines
411 .waitMachine({
412 app_name: appName,
413 machine_id: machineId,
414 state: "destroyed",
415 timeout: WAIT_TIMEOUT_SECONDS,
416 })
417 .pipe(
418 Retry.none,
419 Effect.as(undefined),
420 Effect.catchTag("NotFound", () => Effect.void),
421 Effect.retry({
422 times: 6,
423 schedule: waitBackoff,
424 while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
425 }),
426 Effect.timeout("50 seconds"),
427 );
428
429 export const ensureStarted = (
430 appName: string,
431 machine: FlyMachine,
432 skipLaunch: boolean,
433 healthTimeoutMs = 60_000,
434 expectedConfig?: FlyMachineConfig,
435 existingLeases?: MachineLeases,
436 ) =>
437 usingMachineLeases(appName, existingLeases, (leases) =>
438 ensureLeasedStarted(appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases),
439 );
440
441 const ensureLeasedStarted = Effect.fn(function* (
442 appName: string,
443 machine: FlyMachine,
444 skipLaunch: boolean,
445 healthTimeoutMs: number,
446 expectedConfig: FlyMachineConfig | undefined,
447 leases: MachineLeases,
448 ) {
449 const machineId = machine.id;
450 if (machineId === undefined || skipLaunch) return machine;
451 yield* leases.acquire([machineId]);
452 const started = yield* Effect.gen(function* () {
453 // Create/update responses can lag Fly's automatic launch.
454 const current = yield* getLeasedMachine(appName, machineId);
455 if (!sameOwnership(current, machine)) {
456 return yield* new ReplicaOwnershipChanged({ appName, machineId });
457 }
458 yield* Effect.logDebug("Fly machine startup", { appName, machineId, state: current.state });
459 if (
460 current.state === "stopped" ||
461 current.state === "suspended" ||
462 current.state === "failed"
463 ) {
464 yield* leases.mutate(machineId, (lease_nonce) =>
465 machines.startMachine({ app_name: appName, machine_id: machineId, lease_nonce }),
466 );
467 }
468 // Re-observe state between waits instead of retrying the wait in the SDK.
469 yield* machines
470 .waitMachine({
471 app_name: appName,
472 machine_id: machineId,
473 state: "started",
474 instance_id: current.instance_id,
475 timeout: WAIT_TIMEOUT_SECONDS,
476 })
477 .pipe(Retry.none);
478 return yield* getLeasedMachine(appName, machineId);
479 }).pipe(
480 Effect.retry({
481 times: 6,
482 schedule: waitBackoff,
483 while: (error) =>
484 error._tag === "MachineStartFromCreatedState" ||
485 error._tag === "MachineReplacing" ||
486 error._tag === "MachineWaitTimeout" ||
487 error._tag === "Conflict" ||
488 error._tag === "GatewayTimeout",
489 }),
490 Effect.timeout("180 seconds"),
491 );
492 return yield* waitHealthy(appName, started, healthTimeoutMs, expectedConfig ?? machine.config);
493 });
494
495 export const deleteMachine = (appName: string, machineId: string, existingLeases?: MachineLeases) =>
496 usingMachineLeases(appName, existingLeases, (leases) =>
497 Effect.gen(function* () {
498 if (appName.length === 0 || machineId.length === 0) return;
499 yield* leases.acquire([machineId]);
500 yield* leases.remove(machineId, (lease_nonce) =>
501 machines
502 .deleteMachine({ app_name: appName, machine_id: machineId, force: true, lease_nonce })
503 .pipe(Effect.timeout("30 seconds")),
504 );
505 yield* waitDestroyed(appName, machineId);
506 }).pipe(Effect.catchTag("NotFound", () => leases.forget(machineId))),
507 );
508
509 const durationNanoseconds = (value: string | undefined) => {
510 if (value === undefined || value === "") return undefined;
511 const negative = value.startsWith("-");
512 const duration = /^[+-]/.test(value) ? value.slice(1) : value;
513 if (duration === "0") return 0n;
514 const limit = negative ? 1n << 63n : (1n << 63n) - 1n;
515 const units: Record<string, bigint> = {
516 ns: 1n,
517 us: 1_000n,
518 µs: 1_000n,
519 μs: 1_000n,
520 ms: 1_000_000n,
521 s: 1_000_000_000n,
522 m: 60_000_000_000n,
523 h: 3_600_000_000_000n,
524 };
525 let total = 0n;
526 let consumed = 0;
527 for (const part of duration.matchAll(/(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gy)) {
528 const [integer = "", fraction = ""] = part[1]!.split(".");
529 const whole = integer.replace(/^0+/, "") || "0";
530 if (whole.length > 19) return undefined;
531 const unit = units[part[2]!]!;
532 let fractionalNanos = 0n;
533 // Truncate each component to nanoseconds without unbounded BigInt operands.
534 for (let index = fraction.length - 1; index >= 0; index--) {
535 fractionalNanos = (BigInt(fraction[index]!) * unit + fractionalNanos) / 10n;
536 }
537 total += BigInt(whole) * unit + fractionalNanos;
538 if (total > limit) return undefined;
539 consumed += part[0].length;
540 }
541 if (consumed === 0 || consumed !== duration.length) return undefined;
542 return negative ? -total : total;
543 };
544
545 const stopTimeoutMillis = (timeout: string | undefined) => {
546 if (timeout === "") return 0;
547 // Shutdown policies retain their unsigned millisecond-or-larger syntax.
548 if (timeout === undefined || !/^(?:\d+(?:\.\d+)?(?:ms|s|m|h))+$/.test(timeout)) {
549 return undefined;
550 }
551 const nanos = durationNanoseconds(timeout);
552 return nanos === undefined ? undefined : Number(nanos) / 1_000_000;
553 };
554
555 export const sameStopConfig = (
556 observed: FlyStopConfig | undefined,
557 desired: FlyStopConfig | undefined,
558 ) =>
559 observed?.signal === desired?.signal &&
560 stopTimeoutMillis(observed?.timeout) === stopTimeoutMillis(desired?.timeout);
561
562 export class ShutdownPolicyMismatch extends Data.TaggedError("Fly.ShutdownPolicyMismatch")<{
563 machineId: string;
564 message: string;
565 }> {}
566
567 export const predecessorShutdown = Effect.fn(function* (machine: FlyMachine) {
568 const persisted = machine.config?.stop_config;
569 const injected = machine.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS;
570 const timeoutMs = stopTimeoutMillis(persisted?.timeout);
571 if (injected !== undefined) {
572 const managedTimeout = Number(injected);
573 const signal = persisted?.signal ?? "SIGTERM";
574 if (
575 !/^\d+$/.test(injected) ||
576 !Number.isSafeInteger(managedTimeout) ||
577 managedTimeout <= 0 ||
578 managedTimeout > 300_000 ||
579 (signal !== "SIGTERM" && signal !== "SIGINT") ||
580 (persisted?.timeout !== undefined && timeoutMs !== managedTimeout)
581 ) {
582 return yield* new ShutdownPolicyMismatch({
583 machineId: machine.id ?? "",
584 message:
585 "The predecessor's managed shutdown environment and stop_config disagree; reconcile its shutdown policy before migration.",
586 });
587 }
588 return {
589 signal,
590 timeout: persisted?.timeout ?? `${managedTimeout}ms`,
591 timeoutMs: managedTimeout,
592 };
593 }
594 if (persisted?.timeout !== undefined && (!timeoutMs || timeoutMs > 300_000)) {
595 return yield* new ShutdownPolicyMismatch({
596 machineId: machine.id ?? "",
597 message:
598 "The predecessor has an invalid stop_config timeout; refusing to shorten its shutdown window.",
599 });
600 }
601 // Unspecified raw-image overrides must remain unspecified for Fly's defaults.
602 return {
603 signal: persisted?.signal,
604 timeout: persisted?.timeout,
605 timeoutMs: timeoutMs ?? 300_000,
606 };
607 });
608
609 export class ReplicaOwnershipChanged extends Data.TaggedError("Fly.ReplicaOwnershipChanged")<{
610 appName: string;
611 machineId: string;
612 }> {}
613
614 export class ReplicaRetirementIncomplete extends Data.TaggedError(
615 "Fly.ReplicaRetirementIncomplete",
616 )<{ appName: string; residuals: Array<{ machineId: string; stage: string }> }> {}
617
618 const retirementStep =
619 (appName: string, machineId: string, stage: string) =>
620 <A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
621 effect.pipe(
622 Effect.mapError((error) =>
623 error._tag === "NotFound"
624 ? error
625 : new ReplicaRetirementIncomplete({
626 appName,
627 residuals: [{ machineId, stage: `${stage}: ${error._tag}` }],
628 }),
629 ),
630 );
631
632 const sameOwnership = (observed: FlyMachine, snapshot: FlyMachine) =>
633 observed.id === snapshot.id &&
634 [
635 alchemyMetadataKeys.stack,
636 alchemyMetadataKeys.stage,
637 alchemyMetadataKeys.id,
638 alchemyMetadataKeys.type,
639 alchemyMetadataKeys.instance,
640 alchemyMetadataKeys.fqn,
641 alchemyMetadataKeys.generation,
642 ].every((key) => observed.config?.metadata?.[key] === snapshot.config?.metadata?.[key]);
643
644 export const leaseSnapshot = Effect.fn(function* (
645 appName: string,
646 snapshot: FlyMachine[],
647 leases: MachineLeases,
648 ) {
649 const ordered = [...snapshot].sort((a, b) => (a.id ?? "").localeCompare(b.id ?? ""));
650 const leased = new Set<string>();
651 yield* Effect.forEach(
652 ordered,
653 (machine) =>
654 Effect.gen(function* () {
655 if (!machine.id || machine.host_status === "unreachable") return;
656 const id = machine.id;
657 yield* leases.acquire([id]).pipe(
658 Effect.tap(() => Effect.sync(() => leased.add(id))),
659 // A just-created Machine can 404 on the lease API while it already
660 // lists; a Machine that is really gone is dropped by the GET below.
661 Effect.retry({
662 while: (error) => error._tag === "NotFound",
663 schedule: Schedule.spaced("1 second"),
664 times: 5,
665 }),
666 Effect.catchTag("NotFound", () => Effect.void),
667 );
668 }),
669 { concurrency: 1 },
670 ).pipe(Effect.timeout("30 seconds"));
671 const observed: FlyMachine[] = [];
672 for (const machine of ordered) {
673 if (!machine.id) continue;
674 if (machine.host_status === "unreachable") {
675 observed.push(machine);
676 continue;
677 }
678 const current = yield* leased.has(machine.id)
679 ? getLeasedMachineById(appName, machine.id)
680 : getMachineById(appName, machine.id);
681 if (!current) {
682 yield* leases.forget(machine.id);
683 continue;
684 }
685 if (!sameOwnership(current, machine))
686 return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
687 yield* leases.checkTarget(machine.id);
688 observed.push(current);
689 }
690 yield* leases.check;
691 return observed;
692 });
693
694 export const retireMachines = (
695 appName: string,
696 snapshot: FlyMachine[],
697 replacementReady = false,
698 existingLeases?: MachineLeases,
699 ) =>
700 usingMachineLeases(appName, existingLeases, (leases) =>
701 retireLeasedMachines(appName, snapshot, replacementReady, leases),
702 );
703
704 const retireLeasedMachines = Effect.fn(function* (
705 appName: string,
706 snapshot: FlyMachine[],
707 replacementReady: boolean,
708 leases: MachineLeases,
709 ) {
710 const targets = yield* leaseSnapshot(appName, snapshot, leases);
711 const results = yield* Effect.forEach(
712 targets,
713 (machine) =>
714 Effect.gen(function* () {
715 yield* leases.check;
716 const result = yield* Effect.gen(function* () {
717 if (!machine.id) return;
718 if (replacementReady && machine.host_status === "unreachable") {
719 const metadata = machine.config?.metadata;
720 if (
721 !machine.config ||
722 machine.incomplete_config ||
723 (machine.config.mounts?.length ?? 0) > 0 ||
724 !metadata?.[alchemyMetadataKeys.instance] ||
725 !metadata[alchemyMetadataKeys.fqn]
726 ) {
727 return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
728 }
729 yield* Effect.logWarning(
730 "Force-retiring owned stateless Fly Machine on unreachable host; work was not proven drained",
731 { appName, machineId: machine.id },
732 );
733 const machineId = machine.id;
734 const heldNonce = yield* leases.nonceIfHeld(machineId);
735 yield* Effect.gen(function* () {
736 const remove = (lease_nonce?: string) =>
737 machines
738 .deleteMachine({
739 app_name: appName,
740 machine_id: machineId,
741 force: true,
742 lease_nonce,
743 })
744 .pipe(Retry.none, Effect.timeout("30 seconds"));
745 if (heldNonce !== undefined) yield* leases.remove(machineId, remove);
746 else yield* leases.guard(remove());
747 }).pipe(
748 Effect.catchTag("NotFound", () => Effect.void),
749 retirementStep(appName, machineId, "unreachable force-delete"),
750 );
751 yield* leases.forget(machine.id);
752 return yield* waitDestroyed(appName, machine.id);
753 }
754 yield* retireMachine(appName, machine.id, machine, leases);
755 }).pipe(Effect.result);
756 yield* leases.check;
757 return Result.isFailure(result)
758 ? result.failure._tag === "Fly.ReplicaRetirementIncomplete"
759 ? result.failure.residuals
760 : [{ machineId: machine.id ?? "", stage: result.failure._tag }]
761 : [];
762 }),
763 { concurrency: 4 },
764 );
765 const residuals = results.flat();
766 if (residuals.length) return yield* new ReplicaRetirementIncomplete({ appName, residuals });
767 });
768
769 export const retireMachine = (
770 appName: string,
771 machineId: string,
772 snapshot?: FlyMachine,
773 existingLeases?: MachineLeases,
774 ) =>
775 usingMachineLeases(appName, existingLeases, (leases) =>
776 retireLeasedMachine(appName, machineId, snapshot, leases).pipe(
777 Effect.tap(() => leases.forget(machineId)),
778 ),
779 );
780
781 const retireLeasedMachine = Effect.fn(
782 function* (
783 appName: string,
784 machineId: string,
785 snapshot: FlyMachine | undefined,
786 leases: MachineLeases,
787 ) {
788 yield* leases.acquire([machineId]);
789 const current = yield* getMachineById(appName, machineId);
790 if (current === undefined) return;
791 if (snapshot && !sameOwnership(current, snapshot)) {
792 return yield* new ReplicaOwnershipChanged({ appName, machineId });
793 }
794 const stop = yield* predecessorShutdown(current);
795 if (
796 current.cordoned === true &&
797 current.config?.metadata?.[alchemyMetadataKeys.phase] === "candidate"
798 )
799 return yield* deleteMachine(appName, machineId, leases);
800 if (current.state !== "stopped" && current.state !== "created") {
801 yield* leases
802 .mutate(
803 machineId,
804 (lease_nonce) =>
805 machines
806 .cordonMachine({ app_name: appName, machine_id: machineId, lease_nonce })
807 .pipe(Effect.timeout("30 seconds")),
808 { idempotent: true },
809 )
810 .pipe(retirementStep(appName, machineId, "cordon"));
811 if (hasPublishedService(current.config?.services)) yield* Effect.sleep("10 seconds");
812 yield* leases
813 .mutate(
814 machineId,
815 (lease_nonce) =>
816 machines
817 .stopMachine({
818 app_name: appName,
819 machine_id: machineId,
820 lease_nonce,
821 signal: stop.signal,
822 timeout: stop.timeout,
823 })
824 .pipe(Effect.timeout(stop.timeoutMs + 15_000)),
825 { timeoutMs: stop.timeoutMs + 120_000 },
826 )
827 .pipe(retirementStep(appName, machineId, "stop"));
828 yield* machines
829 .waitMachine({
830 app_name: appName,
831 machine_id: machineId,
832 state: "stopped",
833 instance_id: current.instance_id,
834 timeout: WAIT_TIMEOUT_SECONDS,
835 })
836 .pipe(
837 Retry.none,
838 Effect.retry({
839 times: 10,
840 schedule: Schedule.spaced(Math.max(500, stop.timeoutMs / 10)),
841 while: (error) =>
842 error._tag === "MachineWaitTimeout" || error._tag === "GatewayTimeout",
843 }),
844 Effect.timeout(stop.timeoutMs + 15_000),
845 retirementStep(appName, machineId, "wait-stopped"),
846 );
847 }
848 yield* leases
849 .remove(machineId, (lease_nonce) =>
850 machines
851 .deleteMachine({ app_name: appName, machine_id: machineId, force: false, lease_nonce })
852 .pipe(Effect.timeout("30 seconds")),
853 )
854 .pipe(
855 Effect.catchTag("NotFound", () => Effect.void),
856 retirementStep(appName, machineId, "delete"),
857 );
858 yield* waitDestroyed(appName, machineId).pipe(
859 retirementStep(appName, machineId, "verify-absent"),
860 );
861 },
862 Effect.catchTag("NotFound", () => Effect.void),
863 );
864
865 const mountedDisksOf = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): MountedDisk[] =>
866 (machine.config?.mounts ?? []).flatMap((mount) => {
867 const volumeId = mount.volume;
868 const path = mount.path;
869 if (volumeId === undefined || path === undefined) return [];
870 const volume = volumesById.get(volumeId);
871 return [{ path, volumeId, sizeGb: volume?.size_gb ?? 0, name: volume?.name ?? "" }];
872 });
873
874 export const toReplica = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): Replica => ({
875 machineId: machine.id ?? "",
876 name: machine.name ?? "",
877 region: machine.region ?? "",
878 state: machine.state ?? "",
879 instanceId: machine.instance_id,
880 privateIp: machine.private_ip,
881 imageRef: toImageRef(machine.image_ref),
882 guest: toGuestAttrs(machine.config?.guest),
883 mounts: mountedDisksOf(machine, volumesById),
884 });
885
886 export const toReplicaSet = (
887 replicas: Replica[],
888 appName: string,
889 baseName: string,
890 services?: FlyMachineService[],
891 ): ReplicaSet => {
892 const primary = replicas[0];
893 return {
894 appName,
895 machineId: primary?.machineId ?? "",
896 machineIds: replicas.map((replica) => replica.machineId),
897 name: primary?.name ?? baseName,
898 baseName,
899 region: primary?.region ?? "",
900 regions: [
901 ...new Set(replicas.map((replica) => replica.region).filter((region) => region.length > 0)),
902 ],
903 state: primary?.state ?? "",
904 instanceId: primary?.instanceId,
905 privateIp: primary?.privateIp,
906 imageRef: primary?.imageRef,
907 guest: primary?.guest,
908 services,
909 count: replicas.length,
910 mounts: primary?.mounts ?? [],
911 replicas,
912 };
913 };
914
915 export const ownedReplicas = (
916 listed: FlyMachine[],
917 input: {
918 metadata: Record<string, string>;
919 resourceInstanceId: string;
920 fqn: string;
921 baseName?: string;
922 machineIds?: readonly string[];
923 },
924 ) =>
925 listed.filter((machine) => {
926 const metadata = machine.config?.metadata ?? {};
927 if (!Object.entries(input.metadata).every(([key, value]) => metadata[key] === value))
928 return false;
929 if (metadata[alchemyMetadataKeys.instance] !== undefined) {
930 return (
931 metadata[alchemyMetadataKeys.instance] === input.resourceInstanceId &&
932 metadata[alchemyMetadataKeys.fqn] === input.fqn
933 );
934 }
935 return (
936 (machine.id !== undefined && input.machineIds?.includes(machine.id)) ||
937 (input.baseName !== undefined &&
938 (machine.name === input.baseName ||
939 machine.name === replicaMachineName(input.baseName, replicaIndexOf(machine), 2)))
940 );
941 });
942
943 export const listReplicas = Effect.fn(function* (input: {
944 appName: string;
945 id: string;
946 type: FlyAlchemyType;
947 }) {
948 const machines = yield* listMachinesByApp(input.appName);
949 return machines
950 .filter((machine) => isOwnedType(machine, input.type) && alchemyIdOf(machine) === input.id)
951 .sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right));
952 });
953
954 export const listReplicaSets = Effect.fn(function* (type: FlyAlchemyType) {
955 const apps = yield* listOwnedApps();
956 const groups = yield* Effect.forEach(
957 apps,
958 (app) =>
959 listMachinesByApp(app.appName).pipe(
960 Effect.map((machines) => {
961 const owned = machines.filter((machine) => isOwnedType(machine, type));
962 const byId = new Map<string, FlyMachine[]>();
963 for (const machine of owned) {
964 const id = alchemyIdOf(machine);
965 if (id === undefined) continue;
966 const metadata = machine.config?.metadata;
967 const key = JSON.stringify([
968 metadata?.[alchemyMetadataKeys.stack],
969 metadata?.[alchemyMetadataKeys.stage],
970 metadata?.[alchemyMetadataKeys.fqn] ?? id,
971 metadata?.[alchemyMetadataKeys.instance],
972 metadata?.[alchemyMetadataKeys.generation],
973 ]);
974 const group = byId.get(key) ?? [];
975 group.push(machine);
976 byId.set(key, group);
977 }
978 return [...byId.values()].map((group) => {
979 const sorted = [...group].sort(
980 (left, right) => replicaIndexOf(left) - replicaIndexOf(right),
981 );
982 const replicas = sorted.map((machine) => toReplica(machine, new Map()));
983 return toReplicaSet(
984 replicas,
985 app.appName,
986 replicas[0]?.name ?? "",
987 sorted[0]?.config?.services,
988 );
989 });
990 }),
991 ),
992 { concurrency: 8 },
993 );
994 return groups.flat();
995 });
996
997 const pickVolume = (
998 group: FlyVolume[],
999 used: Set<string>,
1000 preferId: string | undefined,
1001 ): FlyVolume | undefined => {
1002 if (preferId !== undefined && !used.has(preferId)) {
1003 const preferred = group.find((volume) => volume.id === preferId);
1004 if (preferred !== undefined) return preferred;
1005 }
1006 return group.find((volume) => volume.id !== undefined && !used.has(volume.id));
1007 };
1008
1009 export interface ReconcileReplicasInput {
1010 fqn: string;
1011 resourceInstanceId: string;
1012 policy: DeploymentPolicy;
1013 checks?: Record<string, MachineCheck>;
1014 id: string;
1015 type: FlyAlchemyType;
1016 appName: string;
1017 baseName: string;
1018 /**
1019 * Regions to run in. `count` is the total replica count; replica `i`
1020 * runs in `regions[i % regions.length]`.
1021 */
1022 regions: readonly string[];
1023 count: number;
1024 disks: DiskSpec[];
1025 skipLaunch?: boolean;
1026 minSecretsVersion?: number;
1027 outputMachineIds?: readonly string[];
1028 preferVolumeIds?: ReadonlyArray<ReadonlyArray<string>>;
1029 configDrifted: (
1030 machine: FlyMachine,
1031 desired: { mounts: FlyMachineMount[]; metadata: Record<string, string> },
1032 ) => boolean;
1033 buildConfig: (replica: {
1034 index: number;
1035 mounts: FlyMachineMount[];
1036 metadata: Record<string, string>;
1037 }) => FlyMachineConfig;
1038 }
1039
1040 export const reconcileReplicas = (input: ReconcileReplicasInput) =>
1041 usingMachineLeases(input.appName, undefined, (leases) => reconcileLeasedReplicas(input, leases));
1042
1043 const reconcileLeasedReplicas = Effect.fn(function* (
1044 input: ReconcileReplicasInput,
1045 leases: MachineLeases,
1046 ) {
1047 const ownership = yield* createMachineMetadata(input.id, input.type);
1048 const alchemy = {
1049 ...ownership,
1050 [alchemyMetadataKeys.instance]: input.resourceInstanceId,
1051 [alchemyMetadataKeys.fqn]: input.fqn,
1052 [alchemyMetadataKeys.baseName]: input.baseName,
1053 };
1054 const buildConfig: ReconcileReplicasInput["buildConfig"] = (replica) => ({
1055 ...input.buildConfig(replica),
1056 checks:
1057 input.checks === undefined
1058 ? undefined
1059 : Object.fromEntries(
1060 Object.entries(input.checks).map(([name, check]) => [name, toFlyServiceCheck(check)]),
1061 ),
1062 stop_config:
1063 input.policy.shutdown === undefined
1064 ? undefined
1065 : { signal: input.policy.shutdown.signal, timeout: input.policy.shutdown.timeout },
1066 });
1067 const config = buildConfig({ index: 0, mounts: [], metadata: alchemy });
1068 yield* validateDeployment(input.policy, config, input.disks.length > 0, input.skipLaunch);
1069 if (input.policy.bluegreen)
1070 return yield* reconcileBlueGreen({ ...input, buildConfig }, ownership, alchemy, leases);
1071 const desiredNames = new Set(
1072 Array.from({ length: input.count }, (_, index) =>
1073 replicaMachineName(input.baseName, index, input.count),
1074 ),
1075 );
1076 const listed = yield* listMachinesByApp(input.appName);
1077 const owned = yield* leaseSnapshot(
1078 input.appName,
1079 ownedReplicas(listed, { ...input, metadata: ownership, machineIds: input.outputMachineIds }),
1080 leases,
1081 );
1082 const byIndex = new Map<number, FlyMachine>();
1083 const preferIds = new Set((input.outputMachineIds ?? []).filter((id) => id.length > 0));
1084 for (const machine of owned) {
1085 const id = machine.id;
1086 if (id !== undefined && preferIds.has(id)) {
1087 byIndex.set(replicaIndexOf(machine), machine);
1088 }
1089 }
1090 for (const machine of owned) {
1091 const name = machine.name;
1092 if (
1093 name === undefined ||
1094 (!desiredNames.has(name) &&
1095 machine.config?.metadata?.[alchemyMetadataKeys.instance] !== input.resourceInstanceId)
1096 )
1097 continue;
1098 const index = replicaIndexOf(machine);
1099 if (!byIndex.has(index)) byIndex.set(index, machine);
1100 }
1101
1102 const scaleDown = [...byIndex].filter(([index]) => index >= input.count);
1103 yield* retireMachines(
1104 input.appName,
1105 scaleDown.map(([, machine]) => machine),
1106 false,
1107 leases,
1108 );
1109 for (const [index] of scaleDown) byIndex.delete(index);
1110
1111 // Volumes are regional: one group per disk and region, sized to the
1112 // replicas that run there.
1113 const replicasIn = (region: string) =>
1114 Array.from({ length: input.count }, (_, index) => index).filter(
1115 (index) => regionOfReplica(input.regions, index) === region,
1116 ).length;
1117 const groups: Array<{
1118 disk: DiskSpec;
1119 name: string;
1120 byRegion: Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>;
1121 }> = [];
1122 for (const [diskIndex, disk] of input.disks.entries()) {
1123 const name = yield* volumeGroupName(input.id, disk);
1124 const preferIds = (input.preferVolumeIds ?? [])
1125 .map((replica) => replica[diskIndex])
1126 .filter((id): id is string => id !== undefined && id.length > 0);
1127 const byRegion = new Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>();
1128 for (const region of input.regions) {
1129 byRegion.set(
1130 region,
1131 yield* ensureVolumeGroup({
1132 appName: input.appName,
1133 name,
1134 region,
1135 count: replicasIn(region),
1136 disk,
1137 preferIds,
1138 }),
1139 );
1140 }
1141 groups.push({ disk, name, byRegion });
1142 }
1143
1144 const usedVolumeIds = new Set<string>();
1145 const live: FlyMachine[] = [];
1146 for (let index = 0; index < input.count; index++) {
1147 const name = replicaMachineName(input.baseName, index, input.count);
1148 const region = regionOfReplica(input.regions, index);
1149 const metadata = {
1150 ...alchemy,
1151 [alchemyMetadataKeys.replica]: String(index),
1152 ...(input.minSecretsVersion === undefined
1153 ? {}
1154 : { [alchemyMetadataKeys.secretsVersion]: String(input.minSecretsVersion) }),
1155 };
1156 const prefer = input.preferVolumeIds?.[index] ?? [];
1157 const mounts: FlyMachineMount[] = [];
1158 for (const [diskIndex, group] of groups.entries()) {
1159 const volume = pickVolume(
1160 group.byRegion.get(region)?.volumes ?? [],
1161 usedVolumeIds,
1162 prefer[diskIndex],
1163 );
1164 const volumeId = volume?.id;
1165 if (volumeId === undefined) {
1166 return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
1167 }
1168 usedVolumeIds.add(volumeId);
1169 mounts.push({ volume: volumeId, path: group.disk.path });
1170 }
1171 const config = buildConfig({ index, mounts, metadata });
1172 let current = byIndex.get(index);
1173 // Fly cannot move a Machine between regions. Machine names are unique
1174 // per App, so retire the old replica before creating it in its region.
1175 if (current !== undefined && current.region !== region) {
1176 yield* retireMachines(input.appName, [current], false, leases);
1177 byIndex.delete(index);
1178 current = undefined;
1179 }
1180 if (current === undefined) {
1181 const created = yield* machines
1182 .createMachine({
1183 app_name: input.appName,
1184 name,
1185 region,
1186 config,
1187 skip_launch: input.skipLaunch === true ? true : undefined,
1188 min_secrets_version: input.minSecretsVersion,
1189 })
1190 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
1191 current =
1192 created ??
1193 (yield* listMachinesByApp(input.appName).pipe(
1194 Effect.map((machines) =>
1195 ownedReplicas(machines, {
1196 ...input,
1197 metadata: ownership,
1198 machineIds: input.outputMachineIds,
1199 }).find((machine) => machine.name === name),
1200 ),
1201 ));
1202 if (current === undefined || current.id === undefined) {
1203 return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
1204 }
1205 const observed = (yield* leaseSnapshot(input.appName, [current], leases))[0];
1206 if (!observed || input.configDrifted(observed, { mounts, metadata })) {
1207 return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
1208 }
1209 current = observed;
1210 } else if (
1211 !sameChecks(current.config?.checks, config.checks) ||
1212 !sameStopConfig(current.config?.stop_config, config.stop_config) ||
1213 input.configDrifted(current, { mounts, metadata })
1214 ) {
1215 const machineId = current.id!;
1216 const currentVersion = current.instance_id;
1217 const previous = current;
1218 const expected = { ...previous, config };
1219 yield* leases.mutate(machineId, (lease_nonce) =>
1220 machines
1221 .updateMachine({
1222 app_name: input.appName,
1223 machine_id: machineId,
1224 current_version: currentVersion,
1225 config,
1226 skip_launch: input.skipLaunch === true ? true : undefined,
1227 min_secrets_version: input.minSecretsVersion,
1228 lease_nonce,
1229 })
1230 .pipe(Effect.timeout("30 seconds")),
1231 );
1232 // Rolling opt-out removes generation metadata; observe that transition before startup.
1233 current = yield* leases.guard(
1234 Effect.gen(function* () {
1235 const observed = yield* machines
1236 .getMachine({ app_name: input.appName, machine_id: machineId })
1237 .pipe(Retry.none);
1238 if (!sameOwnership(observed, expected) && !sameOwnership(observed, previous))
1239 return yield* new ReplicaOwnershipChanged({ appName: input.appName, machineId });
1240 if (
1241 !sameOwnership(observed, expected) ||
1242 input.configDrifted(observed, { mounts, metadata }) ||
1243 !sameChecks(observed.config?.checks, config.checks) ||
1244 !sameStopConfig(observed.config?.stop_config, config.stop_config)
1245 )
1246 return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
1247 return observed;
1248 }).pipe(
1249 Effect.retry({
1250 times: 8,
1251 schedule: waitBackoff,
1252 while: (error) =>
1253 error._tag === "Fly.ReplicaNotCreated" ||
1254 TRANSIENT_GET_TAGS.some((tag) => tag === error._tag),
1255 }),
1256 Effect.timeout("60 seconds"),
1257 ),
1258 );
1259 }
1260
1261 current = yield* ensureStarted(
1262 input.appName,
1263 current,
1264 input.skipLaunch === true,
1265 input.policy.healthTimeoutMs,
1266 config,
1267 leases,
1268 );
1269 if (current.cordoned === true && !input.skipLaunch) {
1270 yield* setRouting(input.appName, current.id!, false, leases);
1271 if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds");
1272 current = yield* waitHealthy(input.appName, current, input.policy.healthTimeoutMs, config);
1273 }
1274 live.push(current);
1275 }
1276
1277 const liveIds = new Set(live.map((machine) => machine.id));
1278 yield* retireMachines(
1279 input.appName,
1280 owned.filter((machine) => !liveIds.has(machine.id)),
1281 false,
1282 leases,
1283 );
1284
1285 for (const group of groups) {
1286 for (const extra of [...group.byRegion.values()].flatMap((placed) => placed.extras)) {
1287 const volumeId = extra.id;
1288 if (volumeId === undefined || usedVolumeIds.has(volumeId)) continue;
1289 yield* deleteVolume(input.appName, volumeId);
1290 }
1291 }
1292
1293 const volumesById = new Map<string, FlyVolume>();
1294 for (const group of groups) {
1295 for (const volume of [...group.byRegion.values()].flatMap((placed) => placed.volumes)) {
1296 if (volume.id !== undefined) volumesById.set(volume.id, volume);
1297 }
1298 }
1299 const fresh = yield* Effect.forEach(
1300 live,
1301 (machine) =>
1302 machine.id === undefined
1303 ? Effect.succeed(machine)
1304 : getMachineById(input.appName, machine.id).pipe(Effect.map((next) => next ?? machine)),
1305 { concurrency: 4 },
1306 );
1307 const replicas = fresh.map((machine) => toReplica(machine, volumesById));
1308 return toReplicaSet(replicas, input.appName, input.baseName, fresh[0]?.config?.services);
1309 });
1310
1311 export const deleteReplicaSet = Effect.fn(function* (input: {
1312 appName: string;
1313 id: string;
1314 type: FlyAlchemyType;
1315 fqn: string;
1316 resourceInstanceId: string;
1317 force?: boolean;
1318 machineIds: readonly string[];
1319 volumeIds: readonly string[];
1320 }) {
1321 return yield* usingMachineLeases(input.appName, undefined, (leases) =>
1322 Effect.gen(function* () {
1323 const owned = yield* leaseSnapshot(
1324 input.appName,
1325 ownedReplicas(yield* listMachinesByApp(input.appName), {
1326 ...input,
1327 metadata: yield* createMachineMetadata(input.id, input.type),
1328 }),
1329 leases,
1330 );
1331 if (input.force) {
1332 yield* Effect.forEach(
1333 owned,
1334 (machine) =>
1335 Effect.gen(function* () {
1336 if (!machine.id) return;
1337 const current = yield* getMachineById(input.appName, machine.id);
1338 if (!current) return;
1339 if (!sameOwnership(current, machine))
1340 return yield* new ReplicaOwnershipChanged({
1341 appName: input.appName,
1342 machineId: machine.id,
1343 });
1344 yield* deleteMachine(input.appName, machine.id, leases);
1345 }),
1346 { concurrency: 4 },
1347 );
1348 } else yield* retireMachines(input.appName, owned, false, leases);
1349 yield* Effect.forEach(
1350 [...new Set(input.volumeIds)].filter((id) => id.length > 0),
1351 (volumeId) => deleteVolume(input.appName, volumeId),
1352 { concurrency: 4 },
1353 );
1354 }),
1355 );
1356 });
1357
1358 export const volumeIdsOf = (set: {
1359 mounts?: readonly MountedDisk[];
1360 replicas?: readonly Replica[];
1361 }): string[] => {
1362 const ids = new Set<string>();
1363 for (const mount of set.mounts ?? []) ids.add(mount.volumeId);
1364 for (const replica of set.replicas ?? []) {
1365 for (const mount of replica.mounts) ids.add(mount.volumeId);
1366 }
1367 return [...ids];
1368 };
1369
1370 export const groupGenerations = (machines: FlyMachine[]) => {
1371 const groups = new Map<string | undefined, FlyMachine[]>();
1372 for (const machine of machines) {
1373 const generation = machine.config?.metadata?.[alchemyMetadataKeys.generation];
1374 const group = groups.get(generation) ?? [];
1375 group.push(machine);
1376 groups.set(generation, group);
1377 }
1378 return groups;
1379 };
1380
1381 export const observeReplicaSet = Effect.fn(function* (input: {
1382 appName?: string;
1383 fqn: string;
1384 resourceInstanceId: string;
1385 id: string;
1386 type: FlyAlchemyType;
1387 machineIds?: readonly string[];
1388 baseName?: string;
1389 }) {
1390 if (input.appName === undefined) return undefined;
1391 const owned = ownedReplicas(yield* listMachinesByApp(input.appName), {
1392 ...input,
1393 metadata: yield* createMachineMetadata(input.id, input.type),
1394 });
1395 const groups = groupGenerations(owned);
1396 const committed = [...groups.entries()]
1397 .filter(([generation, group]) => {
1398 if (generation === undefined) return false;
1399 const count = Number(group[0]?.config?.metadata?.[alchemyMetadataKeys.count]);
1400 if (group.some((machine) => classifyDeploymentState(machine).protocol === "invalid"))
1401 return false;
1402 const protocol2 = group.some(
1403 (machine) => machine.config?.metadata?.[alchemyMetadataKeys.protocol] === "2",
1404 );
1405 if (protocol2 && !validProtocol2Generation(group)) return false;
1406 return (
1407 Number.isSafeInteger(count) &&
1408 count > 0 &&
1409 group.length === count &&
1410 new Set(group.map(replicaIndexOf)).size === count &&
1411 group.every((machine) => {
1412 const metadata = machine.config?.metadata;
1413 const index = replicaIndexOf(machine);
1414 return (
1415 index >= 0 &&
1416 index < count &&
1417 Number(metadata?.[alchemyMetadataKeys.count]) === count &&
1418 metadata?.[alchemyMetadataKeys.phase] === "active" &&
1419 machine.cordoned === false &&
1420 (metadata[alchemyMetadataKeys.protocol] === undefined ||
1421 ((metadata[alchemyMetadataKeys.protocol] === "1" ||
1422 metadata[alchemyMetadataKeys.protocol] === "2") &&
1423 metadata[alchemyMetadataKeys.restored] === "true" &&
1424 (metadata[alchemyMetadataKeys.role] === "idle" ||
1425 (machine.instance_id !== undefined &&
1426 metadata[alchemyMetadataKeys.checkedInstance] === machine.instance_id))))
1427 );
1428 })
1429 );
1430 })
1431 .sort(
1432 ([, a], [, b]) =>
1433 Number(b[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0) -
1434 Number(a[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0),
1435 );
1436 const legacy = (groups.get(undefined) ?? []).filter((machine) => {
1437 const { protocol } = classifyDeploymentState(machine);
1438 return protocol === "legacy" || protocol === "1";
1439 });
1440 const listed = (committed[0]?.[1] ?? legacy).sort(
1441 (a, b) => replicaIndexOf(a) - replicaIndexOf(b),
1442 );
1443 const rolloutPending = owned.some((machine) => !listed.includes(machine));
1444 if (listed.length > 0) {
1445 const volumesById = new Map<string, FlyVolume>();
1446 for (const machine of listed) {
1447 for (const mount of machine.config?.mounts ?? []) {
1448 const volumeId = mount.volume;
1449 if (volumeId === undefined || volumesById.has(volumeId)) continue;
1450 const volume = yield* getVolumeById(input.appName, volumeId);
1451 if (volume !== undefined) volumesById.set(volumeId, volume);
1452 }
1453 }
1454 const replicas = listed.map((machine) => toReplica(machine, volumesById));
1455 return {
1456 ...toReplicaSet(
1457 replicas,
1458 input.appName,
1459 listed[0]?.config?.metadata?.[alchemyMetadataKeys.baseName] ??
1460 input.baseName ??
1461 replicas[0]?.name ??
1462 "",
1463 listed[0]?.config?.services,
1464 ),
1465 rolloutPending,
1466 };
1467 }
1468 if (owned.length)
1469 return {
1470 ...toReplicaSet([], input.appName, input.baseName ?? ""),
1471 region: owned[0]?.region ?? "",
1472 regions: [
1473 ...new Set(
1474 owned.map((machine) => machine.region ?? "").filter((region) => region.length > 0),
1475 ),
1476 ],
1477 rolloutPending: true,
1478 };
1479 return undefined;
1480 });