The best way to control Claude Code, Codex and more from any computer or phone

main
ts 711 lines 25.8 KB
Raw
1 import * as machines from "@distilled.cloud/fly-io/machines";
2 import type { Machine } from "@distilled.cloud/fly-io/machines";
3 import * as Retry from "@distilled.cloud/fly-io/Retry";
4 import { deepEqual } from "alchemy/Diff";
5 import { sha256Object } from "alchemy/Util/sha256";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import {
10 applyImageSet,
11 encodeImageSet,
12 pinsFromConfig,
13 sameImageSet,
14 validObservedImageSet,
15 } from "./DeploymentImages.ts";
16 import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts";
17 import { usingMachineLeases, type MachineLeases } from "./leases.ts";
18 import { canonicalContainers } from "./MachineContainers.ts";
19 import { alchemyMetadataKeys as keys } from "./Metadata.ts";
20 import { regionOfReplica } from "./Region.ts";
21 import {
22 autostopMode,
23 checksPassing,
24 deleteMachine,
25 ensureStarted,
26 getMachineById,
27 groupGenerations,
28 hasPublishedService,
29 listMachinesByApp,
30 leaseSnapshot,
31 ownedReplicas,
32 ReplicaNotCreated,
33 replicaIndexOf,
34 retireMachines,
35 sameChecks,
36 sameServices,
37 sameStopConfig,
38 toReplica,
39 toReplicaSet,
40 waitHealthy,
41 type ReconcileReplicasInput,
42 } from "./replicas.ts";
43
44 const metadataOf = (machine: Machine) =>
45 Object.fromEntries(
46 Object.entries(machine.config?.metadata ?? {}).flatMap(([key, value]) =>
47 value === undefined ? [] : [[key, value]],
48 ),
49 );
50
51 const setMetadata = Effect.fn(function* (
52 appName: string,
53 machine: Machine,
54 metadata: Record<string, string>,
55 leases: MachineLeases,
56 ) {
57 yield* leases.checkTarget(machine.id!);
58 // Metadata is not nonce-fenced by Fly; it is recovery evidence, not a lock.
59 yield* leases.mutate(
60 machine.id!,
61 () =>
62 machines
63 .patchMachineMetadata({ app_name: appName, machine_id: machine.id!, metadata })
64 .pipe(Effect.timeout("30 seconds")),
65 { idempotent: true },
66 );
67 yield* Effect.sync(() => {
68 machine.config = { ...machine.config, metadata };
69 });
70 });
71
72 const phase = Effect.fn(function* (
73 appName: string,
74 machine: Machine,
75 value: string,
76 leases: MachineLeases,
77 ) {
78 if (machine.config?.metadata?.[keys.phase] !== value) {
79 yield* setMetadata(appName, machine, { ...metadataOf(machine), [keys.phase]: value }, leases);
80 }
81 });
82
83 export const setRouting = (
84 appName: string,
85 machineId: string,
86 cordoned: boolean,
87 existingLeases?: MachineLeases,
88 ) =>
89 usingMachineLeases(appName, existingLeases, (leases) =>
90 Effect.gen(function* () {
91 yield* leases.acquire([machineId]);
92 yield* leases.mutate(
93 machineId,
94 (lease_nonce) =>
95 Effect.gen(function* () {
96 const request = { app_name: appName, machine_id: machineId, lease_nonce };
97 if (cordoned) yield* machines.cordonMachine(request);
98 else yield* machines.uncordonMachine(request);
99 }).pipe(Effect.timeout("30 seconds")),
100 { idempotent: true },
101 );
102 yield* getMachineById(appName, machineId).pipe(
103 Effect.filterOrFail(
104 (machine) => machine !== undefined && machine.cordoned === cordoned,
105 () => new ReplicaNotCreated({ appName, resourceName: machineId }),
106 ),
107 Effect.retry({
108 times: 8,
109 schedule: Schedule.spaced("500 millis"),
110 while: (error) => error._tag === "Fly.ReplicaNotCreated",
111 }),
112 );
113 }),
114 );
115
116 const pinnedImage = (machine: Machine) => {
117 const image = machine.image_ref;
118 if (!image?.digest || !image.repository) return undefined;
119 return `${image.registry ? `${image.registry}/` : ""}${image.repository}@${image.digest}`;
120 };
121
122 export class DeploymentRecoveryAmbiguous extends Data.TaggedError(
123 "Fly.DeploymentRecoveryAmbiguous",
124 )<{ appName: string; message: string }> {}
125
126 export const readinessRoles = (
127 config: machines.FlyMachineConfig,
128 count: number,
129 predecessors: Machine[],
130 ): Array<"run" | "idle"> => {
131 const services = config.services ?? [];
132 if (!services.length || services.some((service) => autostopMode(service.autostop) === "off")) {
133 return Array.from({ length: count }, () => "run");
134 }
135 const roles: Array<"run" | "idle"> = Array.from({ length: count }, (_, index) =>
136 predecessors.some((machine) => replicaIndexOf(machine) === index && machine.state === "started")
137 ? "run"
138 : "idle",
139 );
140 const floor = Math.min(
141 count,
142 Math.max(1, ...services.map((service) => service.min_machines_running ?? 0)),
143 );
144 for (let index = 0; roles.filter((role) => role === "run").length < floor; index++)
145 roles[index] = "run";
146 return roles;
147 };
148
149 const idle = (machine: Machine) =>
150 machine.state === "stopped" || machine.state === "suspended" || machine.state === "created";
151
152 /** Cloud metadata records preparation and promotion; replica zero commits a complete set. */
153 export const reconcileBlueGreen = Effect.fn(function* (
154 input: ReconcileReplicasInput,
155 ownership: Record<string, string>,
156 metadata: Record<string, string>,
157 leases: MachineLeases,
158 ) {
159 const config = input.buildConfig({ index: 0, mounts: [], metadata });
160 const containerPins = config.containers === undefined ? undefined : pinsFromConfig(config);
161 const containerMode = config.containers !== undefined;
162 const ambiguous = (message: string) =>
163 new DeploymentRecoveryAmbiguous({ appName: input.appName, message });
164 if (containerMode && containerPins === undefined)
165 return yield* ambiguous("Blue/green requires a complete immutable container image set.");
166 const workload = yield* sha256Object({
167 config: containerMode
168 ? { ...config, containers: canonicalContainers(config.containers) }
169 : config,
170 count: input.count,
171 minSecretsVersion: input.minSecretsVersion,
172 // A single region hashes as before, so existing generations stay current.
173 region: input.regions.length === 1 ? input.regions[0] : [...input.regions],
174 });
175 const observe = listMachinesByApp(input.appName).pipe(
176 Effect.flatMap((listed) => {
177 const owned = ownedReplicas(listed, {
178 ...input,
179 metadata: ownership,
180 machineIds: input.outputMachineIds,
181 });
182 const changed = listed.find(
183 (machine) =>
184 machine.id && input.outputMachineIds?.includes(machine.id) && !owned.includes(machine),
185 );
186 return changed
187 ? Effect.fail(
188 ambiguous(`Cached Machine ${changed.id} no longer has the expected ownership.`),
189 )
190 : Effect.succeed(owned);
191 }),
192 );
193 const snapshot = yield* observe;
194 const owned = yield* leaseSnapshot(input.appName, snapshot, leases);
195 // A vanished snapshot member can have a successor outside this inventory.
196 if (snapshot.some((machine) => !owned.some((current) => current.id === machine.id))) {
197 return yield* ambiguous(
198 "Owned Machine membership changed while acquiring leases; retry with a fresh deployment snapshot.",
199 );
200 }
201 for (const machine of owned) {
202 const state = classifyDeploymentState(machine);
203 if (state.protocol === "invalid")
204 return yield* ambiguous(`Machine ${machine.id}: ${state.reason}`);
205 }
206
207 const idleAllowed =
208 (config.services?.length ?? 0) > 0 &&
209 config.services!.every((service) => autostopMode(service.autostop) !== "off");
210 const preparationServices = config.services?.map((service) => ({ ...service, autostop: "off" }));
211 const mismatchOf = (machine: Machine): string | undefined => {
212 const observed = metadataOf(machine);
213 if (containerMode) {
214 if (
215 observed[keys.protocol] !== "2" ||
216 !validObservedImageSet(machine) ||
217 !sameImageSet(pinsFromConfig(machine.config), containerPins)
218 )
219 return "container image set";
220 } else {
221 const pinned = observed[keys.image];
222 if (pinned === undefined && observed[keys.phase] !== "candidate") return "metadata.image";
223 if (!pinnedImage(machine) || (pinned !== undefined && pinned !== pinnedImage(machine)))
224 return "image_ref";
225 }
226 const secretsVersion = Number(observed[keys.secretsVersion] ?? -1);
227 if (
228 input.minSecretsVersion !== undefined &&
229 (!Number.isSafeInteger(secretsVersion) || secretsVersion < input.minSecretsVersion)
230 )
231 return "metadata.secretsVersion";
232 const temporary =
233 observed[keys.role] === "run" &&
234 observed[keys.restored] === "false" &&
235 sameServices(machine.config?.services, preparationServices);
236 if (observed[keys.workload] !== workload) return "metadata.workload";
237 if (
238 input.configDrifted(
239 {
240 ...machine,
241 config: {
242 ...machine.config,
243 image: config.image,
244 services: temporary ? config.services : machine.config?.services,
245 },
246 },
247 {
248 mounts: [],
249 metadata: {
250 ...Object.fromEntries(
251 Object.entries(observed).filter(([key]) => key.startsWith("alchemy.")),
252 ),
253 ...metadata,
254 },
255 },
256 )
257 )
258 return "config";
259 if (!sameChecks(machine.config?.checks, config.checks)) return "config.checks";
260 if (!sameStopConfig(machine.config?.stop_config, config.stop_config))
261 return "config.stop_config";
262 return undefined;
263 };
264 const matches = (machine: Machine) => mismatchOf(machine) === undefined;
265 const groups = groupGenerations(owned);
266 const sequences = new Set<string>();
267 for (const [generation, group] of groups) {
268 if (generation === undefined) continue;
269 const sequence = group[0]?.config?.metadata?.[keys.sequence];
270 if (
271 !sequence ||
272 !Number.isSafeInteger(Number(sequence)) ||
273 Number(sequence) < 1 ||
274 sequences.has(sequence) ||
275 group.some((machine) => machine.config?.metadata?.[keys.sequence] !== sequence) ||
276 (group.some((machine) => machine.config?.metadata?.[keys.protocol] === "2") &&
277 !validProtocol2Generation(group))
278 ) {
279 return yield* ambiguous(
280 "Owned generations have ambiguous sequence/lineage metadata; preserving all capacity.",
281 );
282 }
283 sequences.add(sequence);
284 }
285 const reusable = [...groups.entries()]
286 .filter(([generation, group]) => generation !== undefined && group.every(matches))
287 .sort(
288 ([, a], [, b]) =>
289 Number(b[0]?.config?.metadata?.[keys.sequence] ?? 0) -
290 Number(a[0]?.config?.metadata?.[keys.sequence] ?? 0),
291 )[0];
292 const generation =
293 reusable?.[0] ??
294 (yield* sha256Object({
295 workload,
296 predecessors: owned
297 .map((machine) => machine.id)
298 .sort((a, b) => ((a ?? "") < (b ?? "") ? -1 : (a ?? "") > (b ?? "") ? 1 : 0)),
299 })).slice(0, 20);
300 const desired = reusable?.[1] ?? [];
301 if (
302 new Set(desired.map(replicaIndexOf)).size !== desired.length ||
303 desired.some((machine) => replicaIndexOf(machine) >= input.count)
304 ) {
305 return yield* ambiguous(
306 "The recovered generation has duplicate or out-of-range replica indices.",
307 );
308 }
309 const sequence =
310 desired[0]?.config?.metadata?.[keys.sequence] ??
311 String(
312 1 +
313 Math.max(
314 0,
315 ...owned.map((machine) => Number(machine.config?.metadata?.[keys.sequence] ?? 0)),
316 ),
317 );
318 const predecessors = owned.filter(
319 (machine) => machine.config?.metadata?.[keys.generation] !== generation,
320 );
321 const recordedRoles = desired[0]?.config?.metadata?.[keys.roles];
322 const roles = recordedRoles?.split(",") ?? readinessRoles(config, input.count, predecessors);
323 if (
324 roles.length !== input.count ||
325 roles.some((role) => role !== "run" && role !== "idle") ||
326 !roles.includes("run") ||
327 desired.some(
328 (machine) =>
329 machine.config?.metadata?.[keys.roles] !== recordedRoles ||
330 ((machine.config?.metadata?.[keys.protocol] === "1" ||
331 machine.config?.metadata?.[keys.protocol] === "2") &&
332 machine.config.metadata[keys.role] !== roles[replicaIndexOf(machine)]),
333 )
334 ) {
335 return yield* ambiguous("The recovered generation has inconsistent readiness roles.");
336 }
337 const complete =
338 desired.length === input.count &&
339 desired.every(
340 (machine) =>
341 machine.config?.metadata?.[keys.phase] === "active" &&
342 machine.config.metadata[keys.restored] === "true" &&
343 (roles[replicaIndexOf(machine)] === "idle" ||
344 (machine.instance_id !== undefined &&
345 machine.config.metadata[keys.checkedInstance] === machine.instance_id)) &&
346 machine.cordoned === false,
347 );
348 if (
349 complete &&
350 !predecessors.length &&
351 desired.every((machine) => machine.state === "started" || (idleAllowed && idle(machine)))
352 ) {
353 return toReplicaSet(
354 [...desired]
355 .sort((a, b) => replicaIndexOf(a) - replicaIndexOf(b))
356 .map((machine) => toReplica(machine, new Map())),
357 input.appName,
358 input.baseName,
359 config.services,
360 );
361 }
362 const candidates: Machine[] = [];
363 const snapshotIds = new Set(owned.map((machine) => machine.id));
364 let image = desired[0] ? pinnedImage(desired[0]) : undefined;
365 const order = Array.from({ length: input.count }, (_, index) => index).sort(
366 (a, b) => Number(roles[b] === "run") - Number(roles[a] === "run") || a - b,
367 );
368 // Creation-time predecessor IDs are diagnostic lineage, not deletion authority.
369 const predecessorIds = predecessors
370 .flatMap((machine) => (machine.id ? [machine.id] : []))
371 .sort()
372 .join(",");
373 yield* Effect.gen(function* () {
374 for (const index of order) {
375 const suffix = `-${generation}-${index}`;
376 const name = `${input.baseName.slice(0, 30 - suffix.length).replace(/-+$/g, "")}${suffix}`;
377 let current = desired.find((machine) => replicaIndexOf(machine) === index);
378 if (current === undefined) {
379 const run = roles[index] === "run";
380 const candidateConfig = input.buildConfig({
381 index,
382 mounts: [],
383 metadata: {
384 ...metadata,
385 [keys.replica]: String(index),
386 [keys.generation]: generation,
387 [keys.workload]: workload,
388 [keys.phase]: "candidate",
389 [keys.sequence]: sequence,
390 [keys.count]: String(input.count),
391 [keys.protocol]: containerMode ? "2" : "1",
392 [keys.roles]: roles.join(","),
393 [keys.role]: roles[index]!,
394 [keys.predecessors]: predecessorIds,
395 [keys.restored]: String(!run || sameServices(config.services, preparationServices)),
396 ...(input.minSecretsVersion === undefined
397 ? {}
398 : { [keys.secretsVersion]: String(input.minSecretsVersion) }),
399 ...(containerPins
400 ? { [keys.containerImageSet]: encodeImageSet(containerPins)! }
401 : image
402 ? { [keys.image]: image }
403 : {}),
404 },
405 });
406 const pinnedConfig = containerPins
407 ? applyImageSet(candidateConfig, containerPins)
408 : candidateConfig;
409 if (pinnedConfig === undefined) return yield* ambiguous("Incomplete candidate image set.");
410 const readback = observe.pipe(
411 Effect.map((listed) =>
412 listed.find(
413 (machine) =>
414 machine.name === name &&
415 machine.config?.metadata?.[keys.generation] === generation &&
416 replicaIndexOf(machine) === index &&
417 matches(machine),
418 ),
419 ),
420 Effect.filterOrFail(
421 (machine) => machine !== undefined,
422 () => new ReplicaNotCreated({ appName: input.appName, resourceName: name }),
423 ),
424 Effect.retry({
425 times: 8,
426 schedule: Schedule.spaced("500 millis"),
427 while: (error) => error._tag === "Fly.ReplicaNotCreated",
428 }),
429 );
430 current = yield* machines
431 .createMachine({
432 app_name: input.appName,
433 name,
434 region: regionOfReplica(input.regions, index),
435 config: {
436 ...pinnedConfig,
437 image: containerPins ? undefined : (image ?? config.image),
438 services: run ? preparationServices : config.services,
439 },
440 skip_launch: !run,
441 skip_service_registration: true,
442 min_secrets_version: input.minSecretsVersion,
443 })
444 .pipe(
445 Retry.none,
446 Effect.timeout("30 seconds"),
447 Effect.catchTag(["Conflict", "GatewayTimeout", "TimeoutError"], () => readback),
448 Effect.catchReason("HttpClientError", "TransportError", () => readback),
449 );
450 }
451 if (!current?.id)
452 return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
453 const leased = (yield* leaseSnapshot(input.appName, [current], leases))[0];
454 if (!leased?.id)
455 return yield* ambiguous(
456 `Candidate ${current.id} disappeared before its lease was acquired.`,
457 );
458 const mismatch = mismatchOf(leased);
459 if (mismatch !== undefined)
460 return yield* ambiguous(
461 `Candidate ${current.id} changed before its lease was acquired. Mismatch: ${mismatch}.`,
462 );
463 current = leased;
464 snapshotIds.add(current.id);
465 if (containerPins) {
466 if (
467 !validObservedImageSet(current) ||
468 !sameImageSet(pinsFromConfig(current.config), containerPins)
469 )
470 return yield* ambiguous(`Container image pin mismatch on ${current.id}.`);
471 } else {
472 const resolvedImage = pinnedImage(current);
473 if (!resolvedImage || (image !== undefined && image !== resolvedImage))
474 return yield* ambiguous(`Image pin mismatch on ${current.id}.`);
475 image = resolvedImage;
476 if (current.config?.metadata?.[keys.image] !== image)
477 yield* setMetadata(
478 input.appName,
479 current,
480 { ...metadataOf(current), [keys.image]: image },
481 leases,
482 );
483 }
484 const checked =
485 current.config?.metadata?.[keys.checkedInstance] === current.instance_id &&
486 current.instance_id !== undefined;
487 if (roles[index] === "run" && !(idleAllowed && checked && idle(current))) {
488 current = yield* ensureStarted(
489 input.appName,
490 current,
491 false,
492 input.policy.healthTimeoutMs,
493 current.config,
494 leases,
495 );
496 } else if (!idle(current)) {
497 current = yield* waitHealthy(
498 input.appName,
499 current,
500 input.policy.healthTimeoutMs,
501 current.config,
502 );
503 }
504 candidates[index] = current;
505 }
506 }).pipe(
507 Effect.catch((error) =>
508 Effect.gen(function* () {
509 const found = (yield* observe).filter(
510 (machine) =>
511 snapshotIds.has(machine.id) &&
512 machine.config?.metadata?.[keys.generation] === generation,
513 );
514 // A possibly promoted candidate is capacity, not rollback garbage.
515 if (
516 found.every(
517 (machine) =>
518 (!containerMode || matches(machine)) &&
519 machine.config?.metadata?.[keys.phase] === "candidate" &&
520 machine.cordoned === true,
521 )
522 ) {
523 yield* Effect.forEach(
524 found,
525 (machine) =>
526 deleteMachine(input.appName, machine.id!, leases).pipe(
527 Effect.catch((cleanup) =>
528 Effect.logWarning("Fly candidate cleanup remains pending", {
529 appName: input.appName,
530 machineId: machine.id,
531 error: cleanup._tag,
532 }),
533 ),
534 ),
535 { concurrency: 4 },
536 );
537 }
538 return yield* error;
539 }),
540 ),
541 );
542
543 const needsPromotion = candidates.some(
544 (machine) => machine.cordoned !== false || machine.config?.metadata?.[keys.phase] !== "active",
545 );
546 if (needsPromotion) {
547 for (const machine of candidates) yield* phase(input.appName, machine, "promoting", leases);
548 // Promotion metadata resets reports; validate the whole set before routing any member.
549 for (const [index, machine] of candidates.entries()) {
550 const current = yield* getMachineById(input.appName, machine.id!);
551 if (!current || !matches(current))
552 return yield* ambiguous(`Candidate ${machine.id} changed before promotion.`);
553 if (current.state === "started") {
554 candidates[index] = yield* waitHealthy(
555 input.appName,
556 current,
557 input.policy.healthTimeoutMs,
558 current.config,
559 );
560 } else if (
561 !idleAllowed ||
562 !idle(current) ||
563 (roles[index] === "run" &&
564 (current.instance_id === undefined ||
565 current.config?.metadata?.[keys.checkedInstance] !== current.instance_id))
566 ) {
567 return yield* ambiguous(`Candidate ${machine.id} has no readiness proof before promotion.`);
568 } else candidates[index] = current;
569 }
570 for (const machine of candidates)
571 if (machine.cordoned !== false) yield* setRouting(input.appName, machine.id!, false, leases);
572 if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds");
573 }
574 for (const [index, machine] of candidates.entries()) {
575 let current = machine;
576 if (current.config?.metadata?.[keys.restored] === "false") {
577 // Restoring services creates a new instance; its predecessor's checks cannot prove readiness.
578 const machineId = current.id!;
579 const currentVersion = current.instance_id;
580 const restoreConfig = input.buildConfig({
581 index,
582 mounts: [],
583 metadata: { ...metadataOf(current), [keys.restored]: "true" },
584 });
585 const restored = containerPins
586 ? applyImageSet(restoreConfig, containerPins)
587 : { ...restoreConfig, image };
588 if (restored === undefined)
589 return yield* ambiguous("Incomplete image set during service restoration.");
590 current = yield* leases.mutate(machineId, (lease_nonce) =>
591 machines
592 .updateMachine({
593 app_name: input.appName,
594 machine_id: machineId,
595 lease_nonce,
596 current_version: currentVersion,
597 config: restored,
598 min_secrets_version: input.minSecretsVersion,
599 })
600 .pipe(Effect.timeout("30 seconds")),
601 );
602 current = yield* ensureStarted(
603 input.appName,
604 current,
605 false,
606 input.policy.healthTimeoutMs,
607 config,
608 leases,
609 );
610 } else {
611 current = (yield* getMachineById(input.appName, current.id!)) ?? current;
612 if (current.instance_id !== machine.instance_id) {
613 current = yield* ensureStarted(
614 input.appName,
615 current,
616 false,
617 input.policy.healthTimeoutMs,
618 config,
619 leases,
620 );
621 }
622 if (
623 current.state !== "started" &&
624 roles[index] === "run" &&
625 current.config?.metadata?.[keys.checkedInstance] !== current.instance_id
626 ) {
627 current = yield* ensureStarted(
628 input.appName,
629 current,
630 false,
631 input.policy.healthTimeoutMs,
632 config,
633 leases,
634 );
635 }
636 }
637 candidates[index] = current;
638 }
639 // Reset reports while still pending; an interrupted validation is not a commit.
640 for (const machine of candidates) yield* phase(input.appName, machine, "validating", leases);
641 for (const [index, machine] of candidates.entries()) {
642 const current = yield* getMachineById(input.appName, machine.id!);
643 if (!current || !matches(current) || current.config?.metadata?.[keys.restored] !== "true")
644 return yield* ambiguous(`Candidate ${machine.id} changed before commitment.`);
645 if (current.state === "started")
646 candidates[index] = yield* waitHealthy(
647 input.appName,
648 current,
649 input.policy.healthTimeoutMs,
650 config,
651 );
652 else if (
653 !idleAllowed ||
654 !idle(current) ||
655 (roles[index] === "run" &&
656 (current.instance_id === undefined ||
657 (current.config?.metadata?.[keys.checkedInstance] !== current.instance_id &&
658 !(machine.instance_id === current.instance_id && checksPassing(machine, config)))))
659 ) {
660 return yield* ambiguous(
661 `Candidate ${machine.id} has no readiness proof for its current instance.`,
662 );
663 } else candidates[index] = current;
664 }
665 // All final checks precede the commit stamps, with replica zero last.
666 for (const machine of [...candidates].reverse()) {
667 const committed = {
668 ...metadataOf(machine),
669 [keys.phase]: "active",
670 ...(machine.state === "started" || roles[replicaIndexOf(machine)] === "run"
671 ? { [keys.checkedInstance]: machine.instance_id! }
672 : {}),
673 };
674 if (!deepEqual(metadataOf(machine), committed))
675 yield* setMetadata(input.appName, machine, committed, leases);
676 }
677 // Legacy ownership is required; predecessor phase stamps are only advisory.
678 for (const machine of predecessors) {
679 if (machine.config?.metadata?.[keys.protocol] === "2") {
680 const observed = yield* getMachineById(input.appName, machine.id!);
681 if (
682 !observed ||
683 observed.config?.metadata?.[keys.protocol] !== "2" ||
684 !validObservedImageSet(observed) ||
685 !sameImageSet(pinsFromConfig(observed.config), pinsFromConfig(machine.config))
686 )
687 return yield* ambiguous(
688 `Predecessor ${machine.id} image identity changed before retirement.`,
689 );
690 }
691 if (machine.config?.metadata?.[keys.instance] === undefined)
692 yield* setMetadata(input.appName, machine, { ...metadataOf(machine), ...metadata }, leases);
693 if (machine.host_status !== "unreachable")
694 yield* phase(input.appName, machine, "retiring", leases).pipe(
695 Effect.catch((error) =>
696 Effect.logWarning("Fly predecessor phase metadata was not updated", {
697 appName: input.appName,
698 machineId: machine.id,
699 error: error._tag,
700 }),
701 ),
702 );
703 }
704 yield* retireMachines(input.appName, predecessors, true, leases);
705 return toReplicaSet(
706 candidates.map((machine) => toReplica(machine, new Map())),
707 input.appName,
708 input.baseName,
709 config.services,
710 );
711 });