The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 2 | import type { Machine } from "@distilled.cloud/fly-io/machines"; |
| 3 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 4 | import { deepEqual } from "alchemy/Diff"; |
| 5 | import { sha256Object } from "alchemy/Util/sha256"; |
| 6 | import * as Data from "effect/Data"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Schedule from "effect/Schedule"; |
| 9 | import { |
| 10 | applyImageSet, |
| 11 | encodeImageSet, |
| 12 | pinsFromConfig, |
| 13 | sameImageSet, |
| 14 | validObservedImageSet, |
| 15 | } from "./DeploymentImages.ts"; |
| 16 | import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts"; |
| 17 | import { usingMachineLeases, type MachineLeases } from "./leases.ts"; |
| 18 | import { canonicalContainers } from "./MachineContainers.ts"; |
| 19 | import { alchemyMetadataKeys as keys } from "./Metadata.ts"; |
| 20 | import { regionOfReplica } from "./Region.ts"; |
| 21 | import { |
| 22 | autostopMode, |
| 23 | checksPassing, |
| 24 | deleteMachine, |
| 25 | ensureStarted, |
| 26 | getMachineById, |
| 27 | groupGenerations, |
| 28 | hasPublishedService, |
| 29 | listMachinesByApp, |
| 30 | leaseSnapshot, |
| 31 | ownedReplicas, |
| 32 | ReplicaNotCreated, |
| 33 | replicaIndexOf, |
| 34 | retireMachines, |
| 35 | sameChecks, |
| 36 | sameServices, |
| 37 | sameStopConfig, |
| 38 | toReplica, |
| 39 | toReplicaSet, |
| 40 | waitHealthy, |
| 41 | type ReconcileReplicasInput, |
| 42 | } from "./replicas.ts"; |
| 43 | |
| 44 | const metadataOf = (machine: Machine) => |
| 45 | Object.fromEntries( |
| 46 | Object.entries(machine.config?.metadata ?? {}).flatMap(([key, value]) => |
| 47 | value === undefined ? [] : [[key, value]], |
| 48 | ), |
| 49 | ); |
| 50 | |
| 51 | const setMetadata = Effect.fn(function* ( |
| 52 | appName: string, |
| 53 | machine: Machine, |
| 54 | metadata: Record<string, string>, |
| 55 | leases: MachineLeases, |
| 56 | ) { |
| 57 | yield* leases.checkTarget(machine.id!); |
| 58 | // Metadata is not nonce-fenced by Fly; it is recovery evidence, not a lock. |
| 59 | yield* leases.mutate( |
| 60 | machine.id!, |
| 61 | () => |
| 62 | machines |
| 63 | .patchMachineMetadata({ app_name: appName, machine_id: machine.id!, metadata }) |
| 64 | .pipe(Effect.timeout("30 seconds")), |
| 65 | { idempotent: true }, |
| 66 | ); |
| 67 | yield* Effect.sync(() => { |
| 68 | machine.config = { ...machine.config, metadata }; |
| 69 | }); |
| 70 | }); |
| 71 | |
| 72 | const phase = Effect.fn(function* ( |
| 73 | appName: string, |
| 74 | machine: Machine, |
| 75 | value: string, |
| 76 | leases: MachineLeases, |
| 77 | ) { |
| 78 | if (machine.config?.metadata?.[keys.phase] !== value) { |
| 79 | yield* setMetadata(appName, machine, { ...metadataOf(machine), [keys.phase]: value }, leases); |
| 80 | } |
| 81 | }); |
| 82 | |
| 83 | export const setRouting = ( |
| 84 | appName: string, |
| 85 | machineId: string, |
| 86 | cordoned: boolean, |
| 87 | existingLeases?: MachineLeases, |
| 88 | ) => |
| 89 | usingMachineLeases(appName, existingLeases, (leases) => |
| 90 | Effect.gen(function* () { |
| 91 | yield* leases.acquire([machineId]); |
| 92 | yield* leases.mutate( |
| 93 | machineId, |
| 94 | (lease_nonce) => |
| 95 | Effect.gen(function* () { |
| 96 | const request = { app_name: appName, machine_id: machineId, lease_nonce }; |
| 97 | if (cordoned) yield* machines.cordonMachine(request); |
| 98 | else yield* machines.uncordonMachine(request); |
| 99 | }).pipe(Effect.timeout("30 seconds")), |
| 100 | { idempotent: true }, |
| 101 | ); |
| 102 | yield* getMachineById(appName, machineId).pipe( |
| 103 | Effect.filterOrFail( |
| 104 | (machine) => machine !== undefined && machine.cordoned === cordoned, |
| 105 | () => new ReplicaNotCreated({ appName, resourceName: machineId }), |
| 106 | ), |
| 107 | Effect.retry({ |
| 108 | times: 8, |
| 109 | schedule: Schedule.spaced("500 millis"), |
| 110 | while: (error) => error._tag === "Fly.ReplicaNotCreated", |
| 111 | }), |
| 112 | ); |
| 113 | }), |
| 114 | ); |
| 115 | |
| 116 | const pinnedImage = (machine: Machine) => { |
| 117 | const image = machine.image_ref; |
| 118 | if (!image?.digest || !image.repository) return undefined; |
| 119 | return `${image.registry ? `${image.registry}/` : ""}${image.repository}@${image.digest}`; |
| 120 | }; |
| 121 | |
| 122 | export class DeploymentRecoveryAmbiguous extends Data.TaggedError( |
| 123 | "Fly.DeploymentRecoveryAmbiguous", |
| 124 | )<{ appName: string; message: string }> {} |
| 125 | |
| 126 | export const readinessRoles = ( |
| 127 | config: machines.FlyMachineConfig, |
| 128 | count: number, |
| 129 | predecessors: Machine[], |
| 130 | ): Array<"run" | "idle"> => { |
| 131 | const services = config.services ?? []; |
| 132 | if (!services.length || services.some((service) => autostopMode(service.autostop) === "off")) { |
| 133 | return Array.from({ length: count }, () => "run"); |
| 134 | } |
| 135 | const roles: Array<"run" | "idle"> = Array.from({ length: count }, (_, index) => |
| 136 | predecessors.some((machine) => replicaIndexOf(machine) === index && machine.state === "started") |
| 137 | ? "run" |
| 138 | : "idle", |
| 139 | ); |
| 140 | const floor = Math.min( |
| 141 | count, |
| 142 | Math.max(1, ...services.map((service) => service.min_machines_running ?? 0)), |
| 143 | ); |
| 144 | for (let index = 0; roles.filter((role) => role === "run").length < floor; index++) |
| 145 | roles[index] = "run"; |
| 146 | return roles; |
| 147 | }; |
| 148 | |
| 149 | const idle = (machine: Machine) => |
| 150 | machine.state === "stopped" || machine.state === "suspended" || machine.state === "created"; |
| 151 | |
| 152 | /** Cloud metadata records preparation and promotion; replica zero commits a complete set. */ |
| 153 | export const reconcileBlueGreen = Effect.fn(function* ( |
| 154 | input: ReconcileReplicasInput, |
| 155 | ownership: Record<string, string>, |
| 156 | metadata: Record<string, string>, |
| 157 | leases: MachineLeases, |
| 158 | ) { |
| 159 | const config = input.buildConfig({ index: 0, mounts: [], metadata }); |
| 160 | const containerPins = config.containers === undefined ? undefined : pinsFromConfig(config); |
| 161 | const containerMode = config.containers !== undefined; |
| 162 | const ambiguous = (message: string) => |
| 163 | new DeploymentRecoveryAmbiguous({ appName: input.appName, message }); |
| 164 | if (containerMode && containerPins === undefined) |
| 165 | return yield* ambiguous("Blue/green requires a complete immutable container image set."); |
| 166 | const workload = yield* sha256Object({ |
| 167 | config: containerMode |
| 168 | ? { ...config, containers: canonicalContainers(config.containers) } |
| 169 | : config, |
| 170 | count: input.count, |
| 171 | minSecretsVersion: input.minSecretsVersion, |
| 172 | // A single region hashes as before, so existing generations stay current. |
| 173 | region: input.regions.length === 1 ? input.regions[0] : [...input.regions], |
| 174 | }); |
| 175 | const observe = listMachinesByApp(input.appName).pipe( |
| 176 | Effect.flatMap((listed) => { |
| 177 | const owned = ownedReplicas(listed, { |
| 178 | ...input, |
| 179 | metadata: ownership, |
| 180 | machineIds: input.outputMachineIds, |
| 181 | }); |
| 182 | const changed = listed.find( |
| 183 | (machine) => |
| 184 | machine.id && input.outputMachineIds?.includes(machine.id) && !owned.includes(machine), |
| 185 | ); |
| 186 | return changed |
| 187 | ? Effect.fail( |
| 188 | ambiguous(`Cached Machine ${changed.id} no longer has the expected ownership.`), |
| 189 | ) |
| 190 | : Effect.succeed(owned); |
| 191 | }), |
| 192 | ); |
| 193 | const snapshot = yield* observe; |
| 194 | const owned = yield* leaseSnapshot(input.appName, snapshot, leases); |
| 195 | // A vanished snapshot member can have a successor outside this inventory. |
| 196 | if (snapshot.some((machine) => !owned.some((current) => current.id === machine.id))) { |
| 197 | return yield* ambiguous( |
| 198 | "Owned Machine membership changed while acquiring leases; retry with a fresh deployment snapshot.", |
| 199 | ); |
| 200 | } |
| 201 | for (const machine of owned) { |
| 202 | const state = classifyDeploymentState(machine); |
| 203 | if (state.protocol === "invalid") |
| 204 | return yield* ambiguous(`Machine ${machine.id}: ${state.reason}`); |
| 205 | } |
| 206 | |
| 207 | const idleAllowed = |
| 208 | (config.services?.length ?? 0) > 0 && |
| 209 | config.services!.every((service) => autostopMode(service.autostop) !== "off"); |
| 210 | const preparationServices = config.services?.map((service) => ({ ...service, autostop: "off" })); |
| 211 | const mismatchOf = (machine: Machine): string | undefined => { |
| 212 | const observed = metadataOf(machine); |
| 213 | if (containerMode) { |
| 214 | if ( |
| 215 | observed[keys.protocol] !== "2" || |
| 216 | !validObservedImageSet(machine) || |
| 217 | !sameImageSet(pinsFromConfig(machine.config), containerPins) |
| 218 | ) |
| 219 | return "container image set"; |
| 220 | } else { |
| 221 | const pinned = observed[keys.image]; |
| 222 | if (pinned === undefined && observed[keys.phase] !== "candidate") return "metadata.image"; |
| 223 | if (!pinnedImage(machine) || (pinned !== undefined && pinned !== pinnedImage(machine))) |
| 224 | return "image_ref"; |
| 225 | } |
| 226 | const secretsVersion = Number(observed[keys.secretsVersion] ?? -1); |
| 227 | if ( |
| 228 | input.minSecretsVersion !== undefined && |
| 229 | (!Number.isSafeInteger(secretsVersion) || secretsVersion < input.minSecretsVersion) |
| 230 | ) |
| 231 | return "metadata.secretsVersion"; |
| 232 | const temporary = |
| 233 | observed[keys.role] === "run" && |
| 234 | observed[keys.restored] === "false" && |
| 235 | sameServices(machine.config?.services, preparationServices); |
| 236 | if (observed[keys.workload] !== workload) return "metadata.workload"; |
| 237 | if ( |
| 238 | input.configDrifted( |
| 239 | { |
| 240 | ...machine, |
| 241 | config: { |
| 242 | ...machine.config, |
| 243 | image: config.image, |
| 244 | services: temporary ? config.services : machine.config?.services, |
| 245 | }, |
| 246 | }, |
| 247 | { |
| 248 | mounts: [], |
| 249 | metadata: { |
| 250 | ...Object.fromEntries( |
| 251 | Object.entries(observed).filter(([key]) => key.startsWith("alchemy.")), |
| 252 | ), |
| 253 | ...metadata, |
| 254 | }, |
| 255 | }, |
| 256 | ) |
| 257 | ) |
| 258 | return "config"; |
| 259 | if (!sameChecks(machine.config?.checks, config.checks)) return "config.checks"; |
| 260 | if (!sameStopConfig(machine.config?.stop_config, config.stop_config)) |
| 261 | return "config.stop_config"; |
| 262 | return undefined; |
| 263 | }; |
| 264 | const matches = (machine: Machine) => mismatchOf(machine) === undefined; |
| 265 | const groups = groupGenerations(owned); |
| 266 | const sequences = new Set<string>(); |
| 267 | for (const [generation, group] of groups) { |
| 268 | if (generation === undefined) continue; |
| 269 | const sequence = group[0]?.config?.metadata?.[keys.sequence]; |
| 270 | if ( |
| 271 | !sequence || |
| 272 | !Number.isSafeInteger(Number(sequence)) || |
| 273 | Number(sequence) < 1 || |
| 274 | sequences.has(sequence) || |
| 275 | group.some((machine) => machine.config?.metadata?.[keys.sequence] !== sequence) || |
| 276 | (group.some((machine) => machine.config?.metadata?.[keys.protocol] === "2") && |
| 277 | !validProtocol2Generation(group)) |
| 278 | ) { |
| 279 | return yield* ambiguous( |
| 280 | "Owned generations have ambiguous sequence/lineage metadata; preserving all capacity.", |
| 281 | ); |
| 282 | } |
| 283 | sequences.add(sequence); |
| 284 | } |
| 285 | const reusable = [...groups.entries()] |
| 286 | .filter(([generation, group]) => generation !== undefined && group.every(matches)) |
| 287 | .sort( |
| 288 | ([, a], [, b]) => |
| 289 | Number(b[0]?.config?.metadata?.[keys.sequence] ?? 0) - |
| 290 | Number(a[0]?.config?.metadata?.[keys.sequence] ?? 0), |
| 291 | )[0]; |
| 292 | const generation = |
| 293 | reusable?.[0] ?? |
| 294 | (yield* sha256Object({ |
| 295 | workload, |
| 296 | predecessors: owned |
| 297 | .map((machine) => machine.id) |
| 298 | .sort((a, b) => ((a ?? "") < (b ?? "") ? -1 : (a ?? "") > (b ?? "") ? 1 : 0)), |
| 299 | })).slice(0, 20); |
| 300 | const desired = reusable?.[1] ?? []; |
| 301 | if ( |
| 302 | new Set(desired.map(replicaIndexOf)).size !== desired.length || |
| 303 | desired.some((machine) => replicaIndexOf(machine) >= input.count) |
| 304 | ) { |
| 305 | return yield* ambiguous( |
| 306 | "The recovered generation has duplicate or out-of-range replica indices.", |
| 307 | ); |
| 308 | } |
| 309 | const sequence = |
| 310 | desired[0]?.config?.metadata?.[keys.sequence] ?? |
| 311 | String( |
| 312 | 1 + |
| 313 | Math.max( |
| 314 | 0, |
| 315 | ...owned.map((machine) => Number(machine.config?.metadata?.[keys.sequence] ?? 0)), |
| 316 | ), |
| 317 | ); |
| 318 | const predecessors = owned.filter( |
| 319 | (machine) => machine.config?.metadata?.[keys.generation] !== generation, |
| 320 | ); |
| 321 | const recordedRoles = desired[0]?.config?.metadata?.[keys.roles]; |
| 322 | const roles = recordedRoles?.split(",") ?? readinessRoles(config, input.count, predecessors); |
| 323 | if ( |
| 324 | roles.length !== input.count || |
| 325 | roles.some((role) => role !== "run" && role !== "idle") || |
| 326 | !roles.includes("run") || |
| 327 | desired.some( |
| 328 | (machine) => |
| 329 | machine.config?.metadata?.[keys.roles] !== recordedRoles || |
| 330 | ((machine.config?.metadata?.[keys.protocol] === "1" || |
| 331 | machine.config?.metadata?.[keys.protocol] === "2") && |
| 332 | machine.config.metadata[keys.role] !== roles[replicaIndexOf(machine)]), |
| 333 | ) |
| 334 | ) { |
| 335 | return yield* ambiguous("The recovered generation has inconsistent readiness roles."); |
| 336 | } |
| 337 | const complete = |
| 338 | desired.length === input.count && |
| 339 | desired.every( |
| 340 | (machine) => |
| 341 | machine.config?.metadata?.[keys.phase] === "active" && |
| 342 | machine.config.metadata[keys.restored] === "true" && |
| 343 | (roles[replicaIndexOf(machine)] === "idle" || |
| 344 | (machine.instance_id !== undefined && |
| 345 | machine.config.metadata[keys.checkedInstance] === machine.instance_id)) && |
| 346 | machine.cordoned === false, |
| 347 | ); |
| 348 | if ( |
| 349 | complete && |
| 350 | !predecessors.length && |
| 351 | desired.every((machine) => machine.state === "started" || (idleAllowed && idle(machine))) |
| 352 | ) { |
| 353 | return toReplicaSet( |
| 354 | [...desired] |
| 355 | .sort((a, b) => replicaIndexOf(a) - replicaIndexOf(b)) |
| 356 | .map((machine) => toReplica(machine, new Map())), |
| 357 | input.appName, |
| 358 | input.baseName, |
| 359 | config.services, |
| 360 | ); |
| 361 | } |
| 362 | const candidates: Machine[] = []; |
| 363 | const snapshotIds = new Set(owned.map((machine) => machine.id)); |
| 364 | let image = desired[0] ? pinnedImage(desired[0]) : undefined; |
| 365 | const order = Array.from({ length: input.count }, (_, index) => index).sort( |
| 366 | (a, b) => Number(roles[b] === "run") - Number(roles[a] === "run") || a - b, |
| 367 | ); |
| 368 | // Creation-time predecessor IDs are diagnostic lineage, not deletion authority. |
| 369 | const predecessorIds = predecessors |
| 370 | .flatMap((machine) => (machine.id ? [machine.id] : [])) |
| 371 | .sort() |
| 372 | .join(","); |
| 373 | yield* Effect.gen(function* () { |
| 374 | for (const index of order) { |
| 375 | const suffix = `-${generation}-${index}`; |
| 376 | const name = `${input.baseName.slice(0, 30 - suffix.length).replace(/-+$/g, "")}${suffix}`; |
| 377 | let current = desired.find((machine) => replicaIndexOf(machine) === index); |
| 378 | if (current === undefined) { |
| 379 | const run = roles[index] === "run"; |
| 380 | const candidateConfig = input.buildConfig({ |
| 381 | index, |
| 382 | mounts: [], |
| 383 | metadata: { |
| 384 | ...metadata, |
| 385 | [keys.replica]: String(index), |
| 386 | [keys.generation]: generation, |
| 387 | [keys.workload]: workload, |
| 388 | [keys.phase]: "candidate", |
| 389 | [keys.sequence]: sequence, |
| 390 | [keys.count]: String(input.count), |
| 391 | [keys.protocol]: containerMode ? "2" : "1", |
| 392 | [keys.roles]: roles.join(","), |
| 393 | [keys.role]: roles[index]!, |
| 394 | [keys.predecessors]: predecessorIds, |
| 395 | [keys.restored]: String(!run || sameServices(config.services, preparationServices)), |
| 396 | ...(input.minSecretsVersion === undefined |
| 397 | ? {} |
| 398 | : { [keys.secretsVersion]: String(input.minSecretsVersion) }), |
| 399 | ...(containerPins |
| 400 | ? { [keys.containerImageSet]: encodeImageSet(containerPins)! } |
| 401 | : image |
| 402 | ? { [keys.image]: image } |
| 403 | : {}), |
| 404 | }, |
| 405 | }); |
| 406 | const pinnedConfig = containerPins |
| 407 | ? applyImageSet(candidateConfig, containerPins) |
| 408 | : candidateConfig; |
| 409 | if (pinnedConfig === undefined) return yield* ambiguous("Incomplete candidate image set."); |
| 410 | const readback = observe.pipe( |
| 411 | Effect.map((listed) => |
| 412 | listed.find( |
| 413 | (machine) => |
| 414 | machine.name === name && |
| 415 | machine.config?.metadata?.[keys.generation] === generation && |
| 416 | replicaIndexOf(machine) === index && |
| 417 | matches(machine), |
| 418 | ), |
| 419 | ), |
| 420 | Effect.filterOrFail( |
| 421 | (machine) => machine !== undefined, |
| 422 | () => new ReplicaNotCreated({ appName: input.appName, resourceName: name }), |
| 423 | ), |
| 424 | Effect.retry({ |
| 425 | times: 8, |
| 426 | schedule: Schedule.spaced("500 millis"), |
| 427 | while: (error) => error._tag === "Fly.ReplicaNotCreated", |
| 428 | }), |
| 429 | ); |
| 430 | current = yield* machines |
| 431 | .createMachine({ |
| 432 | app_name: input.appName, |
| 433 | name, |
| 434 | region: regionOfReplica(input.regions, index), |
| 435 | config: { |
| 436 | ...pinnedConfig, |
| 437 | image: containerPins ? undefined : (image ?? config.image), |
| 438 | services: run ? preparationServices : config.services, |
| 439 | }, |
| 440 | skip_launch: !run, |
| 441 | skip_service_registration: true, |
| 442 | min_secrets_version: input.minSecretsVersion, |
| 443 | }) |
| 444 | .pipe( |
| 445 | Retry.none, |
| 446 | Effect.timeout("30 seconds"), |
| 447 | Effect.catchTag(["Conflict", "GatewayTimeout", "TimeoutError"], () => readback), |
| 448 | Effect.catchReason("HttpClientError", "TransportError", () => readback), |
| 449 | ); |
| 450 | } |
| 451 | if (!current?.id) |
| 452 | return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name }); |
| 453 | const leased = (yield* leaseSnapshot(input.appName, [current], leases))[0]; |
| 454 | if (!leased?.id) |
| 455 | return yield* ambiguous( |
| 456 | `Candidate ${current.id} disappeared before its lease was acquired.`, |
| 457 | ); |
| 458 | const mismatch = mismatchOf(leased); |
| 459 | if (mismatch !== undefined) |
| 460 | return yield* ambiguous( |
| 461 | `Candidate ${current.id} changed before its lease was acquired. Mismatch: ${mismatch}.`, |
| 462 | ); |
| 463 | current = leased; |
| 464 | snapshotIds.add(current.id); |
| 465 | if (containerPins) { |
| 466 | if ( |
| 467 | !validObservedImageSet(current) || |
| 468 | !sameImageSet(pinsFromConfig(current.config), containerPins) |
| 469 | ) |
| 470 | return yield* ambiguous(`Container image pin mismatch on ${current.id}.`); |
| 471 | } else { |
| 472 | const resolvedImage = pinnedImage(current); |
| 473 | if (!resolvedImage || (image !== undefined && image !== resolvedImage)) |
| 474 | return yield* ambiguous(`Image pin mismatch on ${current.id}.`); |
| 475 | image = resolvedImage; |
| 476 | if (current.config?.metadata?.[keys.image] !== image) |
| 477 | yield* setMetadata( |
| 478 | input.appName, |
| 479 | current, |
| 480 | { ...metadataOf(current), [keys.image]: image }, |
| 481 | leases, |
| 482 | ); |
| 483 | } |
| 484 | const checked = |
| 485 | current.config?.metadata?.[keys.checkedInstance] === current.instance_id && |
| 486 | current.instance_id !== undefined; |
| 487 | if (roles[index] === "run" && !(idleAllowed && checked && idle(current))) { |
| 488 | current = yield* ensureStarted( |
| 489 | input.appName, |
| 490 | current, |
| 491 | false, |
| 492 | input.policy.healthTimeoutMs, |
| 493 | current.config, |
| 494 | leases, |
| 495 | ); |
| 496 | } else if (!idle(current)) { |
| 497 | current = yield* waitHealthy( |
| 498 | input.appName, |
| 499 | current, |
| 500 | input.policy.healthTimeoutMs, |
| 501 | current.config, |
| 502 | ); |
| 503 | } |
| 504 | candidates[index] = current; |
| 505 | } |
| 506 | }).pipe( |
| 507 | Effect.catch((error) => |
| 508 | Effect.gen(function* () { |
| 509 | const found = (yield* observe).filter( |
| 510 | (machine) => |
| 511 | snapshotIds.has(machine.id) && |
| 512 | machine.config?.metadata?.[keys.generation] === generation, |
| 513 | ); |
| 514 | // A possibly promoted candidate is capacity, not rollback garbage. |
| 515 | if ( |
| 516 | found.every( |
| 517 | (machine) => |
| 518 | (!containerMode || matches(machine)) && |
| 519 | machine.config?.metadata?.[keys.phase] === "candidate" && |
| 520 | machine.cordoned === true, |
| 521 | ) |
| 522 | ) { |
| 523 | yield* Effect.forEach( |
| 524 | found, |
| 525 | (machine) => |
| 526 | deleteMachine(input.appName, machine.id!, leases).pipe( |
| 527 | Effect.catch((cleanup) => |
| 528 | Effect.logWarning("Fly candidate cleanup remains pending", { |
| 529 | appName: input.appName, |
| 530 | machineId: machine.id, |
| 531 | error: cleanup._tag, |
| 532 | }), |
| 533 | ), |
| 534 | ), |
| 535 | { concurrency: 4 }, |
| 536 | ); |
| 537 | } |
| 538 | return yield* error; |
| 539 | }), |
| 540 | ), |
| 541 | ); |
| 542 | |
| 543 | const needsPromotion = candidates.some( |
| 544 | (machine) => machine.cordoned !== false || machine.config?.metadata?.[keys.phase] !== "active", |
| 545 | ); |
| 546 | if (needsPromotion) { |
| 547 | for (const machine of candidates) yield* phase(input.appName, machine, "promoting", leases); |
| 548 | // Promotion metadata resets reports; validate the whole set before routing any member. |
| 549 | for (const [index, machine] of candidates.entries()) { |
| 550 | const current = yield* getMachineById(input.appName, machine.id!); |
| 551 | if (!current || !matches(current)) |
| 552 | return yield* ambiguous(`Candidate ${machine.id} changed before promotion.`); |
| 553 | if (current.state === "started") { |
| 554 | candidates[index] = yield* waitHealthy( |
| 555 | input.appName, |
| 556 | current, |
| 557 | input.policy.healthTimeoutMs, |
| 558 | current.config, |
| 559 | ); |
| 560 | } else if ( |
| 561 | !idleAllowed || |
| 562 | !idle(current) || |
| 563 | (roles[index] === "run" && |
| 564 | (current.instance_id === undefined || |
| 565 | current.config?.metadata?.[keys.checkedInstance] !== current.instance_id)) |
| 566 | ) { |
| 567 | return yield* ambiguous(`Candidate ${machine.id} has no readiness proof before promotion.`); |
| 568 | } else candidates[index] = current; |
| 569 | } |
| 570 | for (const machine of candidates) |
| 571 | if (machine.cordoned !== false) yield* setRouting(input.appName, machine.id!, false, leases); |
| 572 | if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds"); |
| 573 | } |
| 574 | for (const [index, machine] of candidates.entries()) { |
| 575 | let current = machine; |
| 576 | if (current.config?.metadata?.[keys.restored] === "false") { |
| 577 | // Restoring services creates a new instance; its predecessor's checks cannot prove readiness. |
| 578 | const machineId = current.id!; |
| 579 | const currentVersion = current.instance_id; |
| 580 | const restoreConfig = input.buildConfig({ |
| 581 | index, |
| 582 | mounts: [], |
| 583 | metadata: { ...metadataOf(current), [keys.restored]: "true" }, |
| 584 | }); |
| 585 | const restored = containerPins |
| 586 | ? applyImageSet(restoreConfig, containerPins) |
| 587 | : { ...restoreConfig, image }; |
| 588 | if (restored === undefined) |
| 589 | return yield* ambiguous("Incomplete image set during service restoration."); |
| 590 | current = yield* leases.mutate(machineId, (lease_nonce) => |
| 591 | machines |
| 592 | .updateMachine({ |
| 593 | app_name: input.appName, |
| 594 | machine_id: machineId, |
| 595 | lease_nonce, |
| 596 | current_version: currentVersion, |
| 597 | config: restored, |
| 598 | min_secrets_version: input.minSecretsVersion, |
| 599 | }) |
| 600 | .pipe(Effect.timeout("30 seconds")), |
| 601 | ); |
| 602 | current = yield* ensureStarted( |
| 603 | input.appName, |
| 604 | current, |
| 605 | false, |
| 606 | input.policy.healthTimeoutMs, |
| 607 | config, |
| 608 | leases, |
| 609 | ); |
| 610 | } else { |
| 611 | current = (yield* getMachineById(input.appName, current.id!)) ?? current; |
| 612 | if (current.instance_id !== machine.instance_id) { |
| 613 | current = yield* ensureStarted( |
| 614 | input.appName, |
| 615 | current, |
| 616 | false, |
| 617 | input.policy.healthTimeoutMs, |
| 618 | config, |
| 619 | leases, |
| 620 | ); |
| 621 | } |
| 622 | if ( |
| 623 | current.state !== "started" && |
| 624 | roles[index] === "run" && |
| 625 | current.config?.metadata?.[keys.checkedInstance] !== current.instance_id |
| 626 | ) { |
| 627 | current = yield* ensureStarted( |
| 628 | input.appName, |
| 629 | current, |
| 630 | false, |
| 631 | input.policy.healthTimeoutMs, |
| 632 | config, |
| 633 | leases, |
| 634 | ); |
| 635 | } |
| 636 | } |
| 637 | candidates[index] = current; |
| 638 | } |
| 639 | // Reset reports while still pending; an interrupted validation is not a commit. |
| 640 | for (const machine of candidates) yield* phase(input.appName, machine, "validating", leases); |
| 641 | for (const [index, machine] of candidates.entries()) { |
| 642 | const current = yield* getMachineById(input.appName, machine.id!); |
| 643 | if (!current || !matches(current) || current.config?.metadata?.[keys.restored] !== "true") |
| 644 | return yield* ambiguous(`Candidate ${machine.id} changed before commitment.`); |
| 645 | if (current.state === "started") |
| 646 | candidates[index] = yield* waitHealthy( |
| 647 | input.appName, |
| 648 | current, |
| 649 | input.policy.healthTimeoutMs, |
| 650 | config, |
| 651 | ); |
| 652 | else if ( |
| 653 | !idleAllowed || |
| 654 | !idle(current) || |
| 655 | (roles[index] === "run" && |
| 656 | (current.instance_id === undefined || |
| 657 | (current.config?.metadata?.[keys.checkedInstance] !== current.instance_id && |
| 658 | !(machine.instance_id === current.instance_id && checksPassing(machine, config))))) |
| 659 | ) { |
| 660 | return yield* ambiguous( |
| 661 | `Candidate ${machine.id} has no readiness proof for its current instance.`, |
| 662 | ); |
| 663 | } else candidates[index] = current; |
| 664 | } |
| 665 | // All final checks precede the commit stamps, with replica zero last. |
| 666 | for (const machine of [...candidates].reverse()) { |
| 667 | const committed = { |
| 668 | ...metadataOf(machine), |
| 669 | [keys.phase]: "active", |
| 670 | ...(machine.state === "started" || roles[replicaIndexOf(machine)] === "run" |
| 671 | ? { [keys.checkedInstance]: machine.instance_id! } |
| 672 | : {}), |
| 673 | }; |
| 674 | if (!deepEqual(metadataOf(machine), committed)) |
| 675 | yield* setMetadata(input.appName, machine, committed, leases); |
| 676 | } |
| 677 | // Legacy ownership is required; predecessor phase stamps are only advisory. |
| 678 | for (const machine of predecessors) { |
| 679 | if (machine.config?.metadata?.[keys.protocol] === "2") { |
| 680 | const observed = yield* getMachineById(input.appName, machine.id!); |
| 681 | if ( |
| 682 | !observed || |
| 683 | observed.config?.metadata?.[keys.protocol] !== "2" || |
| 684 | !validObservedImageSet(observed) || |
| 685 | !sameImageSet(pinsFromConfig(observed.config), pinsFromConfig(machine.config)) |
| 686 | ) |
| 687 | return yield* ambiguous( |
| 688 | `Predecessor ${machine.id} image identity changed before retirement.`, |
| 689 | ); |
| 690 | } |
| 691 | if (machine.config?.metadata?.[keys.instance] === undefined) |
| 692 | yield* setMetadata(input.appName, machine, { ...metadataOf(machine), ...metadata }, leases); |
| 693 | if (machine.host_status !== "unreachable") |
| 694 | yield* phase(input.appName, machine, "retiring", leases).pipe( |
| 695 | Effect.catch((error) => |
| 696 | Effect.logWarning("Fly predecessor phase metadata was not updated", { |
| 697 | appName: input.appName, |
| 698 | machineId: machine.id, |
| 699 | error: error._tag, |
| 700 | }), |
| 701 | ), |
| 702 | ); |
| 703 | } |
| 704 | yield* retireMachines(input.appName, predecessors, true, leases); |
| 705 | return toReplicaSet( |
| 706 | candidates.map((machine) => toReplica(machine, new Map())), |
| 707 | input.appName, |
| 708 | input.baseName, |
| 709 | config.services, |
| 710 | ); |
| 711 | }); |