The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import type { |
| 2 | FlyMachineConfig, |
| 3 | FlyMachineMount, |
| 4 | FlyMachineService, |
| 5 | FlyMachineServiceCheck, |
| 6 | FlyStopConfig, |
| 7 | ImageRef as FlyImageRef, |
| 8 | Machine as FlyMachine, |
| 9 | Volume as FlyVolume, |
| 10 | } from "@distilled.cloud/fly-io/machines"; |
| 11 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 12 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 13 | import { deepEqual } from "alchemy/Diff"; |
| 14 | import * as Data from "effect/Data"; |
| 15 | import * as Effect from "effect/Effect"; |
| 16 | import * as Result from "effect/Result"; |
| 17 | import * as Schedule from "effect/Schedule"; |
| 18 | import { listOwnedApps } from "./App.ts"; |
| 19 | import { reconcileBlueGreen, setRouting } from "./bluegreen.ts"; |
| 20 | import { validateDeployment, type DeploymentPolicy, type MachineCheck } from "./Deployment.ts"; |
| 21 | import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts"; |
| 22 | import { usingMachineLeases, type MachineLeases } from "./leases.ts"; |
| 23 | import type { |
| 24 | MachineGuest, |
| 25 | MachineImageRef, |
| 26 | MachineService, |
| 27 | MachineServiceCheck, |
| 28 | } from "./Machine.ts"; |
| 29 | import { |
| 30 | alchemyMetadataKeys, |
| 31 | createMachineMetadata, |
| 32 | isAlchemyOwnedMetadata, |
| 33 | sanitizeFlyAppName, |
| 34 | type FlyAlchemyType, |
| 35 | } from "./Metadata.ts"; |
| 36 | import type { DiskSpec, MountedDisk } from "./MountVolume.ts"; |
| 37 | import { regionOfReplica } from "./Region.ts"; |
| 38 | import { deleteVolume, ensureVolumeGroup, getVolumeById, volumeGroupName } from "./Volume.ts"; |
| 39 | |
| 40 | const WAIT_TIMEOUT_SECONDS = 8; |
| 41 | const waitBackoff = Schedule.exponential("500 millis"); |
| 42 | const SERVICE_CHECK_NAME_PREFIX = "servicecheck-"; |
| 43 | |
| 44 | export class ReplicaNotCreated extends Data.TaggedError("Fly.ReplicaNotCreated")<{ |
| 45 | resourceName: string; |
| 46 | appName: string; |
| 47 | }> {} |
| 48 | |
| 49 | export class ReplicaChecksNotPassing extends Data.TaggedError("Fly.ReplicaChecksNotPassing")<{ |
| 50 | appName: string; |
| 51 | machineId: string; |
| 52 | checks: ReadonlyArray<{ |
| 53 | name: string | undefined; |
| 54 | status: string | undefined; |
| 55 | output: string | undefined; |
| 56 | }>; |
| 57 | }> { |
| 58 | get message() { |
| 59 | const checks = this.checks.map( |
| 60 | (check) => |
| 61 | `${check.name ?? "unnamed"}: ${check.status ?? "unknown"}${check.output ? ` (${check.output})` : ""}`, |
| 62 | ); |
| 63 | return `Service checks did not pass for ${this.appName}/${this.machineId}: ${checks.join("; ") || "no service check results"}`; |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | export interface Replica { |
| 68 | machineId: string; |
| 69 | name: string; |
| 70 | baseName?: string; |
| 71 | region: string; |
| 72 | state: string; |
| 73 | instanceId: string | undefined; |
| 74 | privateIp: string | undefined; |
| 75 | imageRef: MachineImageRef | undefined; |
| 76 | guest: MachineGuest | undefined; |
| 77 | mounts: MountedDisk[]; |
| 78 | } |
| 79 | |
| 80 | export interface ReplicaSet { |
| 81 | rolloutPending?: boolean; |
| 82 | appName: string; |
| 83 | machineId: string; |
| 84 | machineIds: string[]; |
| 85 | name: string; |
| 86 | baseName?: string; |
| 87 | /** Region of replica 0. */ |
| 88 | region: string; |
| 89 | /** Regions the replicas run in, in replica order. */ |
| 90 | regions: string[]; |
| 91 | state: string; |
| 92 | instanceId: string | undefined; |
| 93 | privateIp: string | undefined; |
| 94 | imageRef: MachineImageRef | undefined; |
| 95 | guest: MachineGuest | undefined; |
| 96 | /** Observed proxy services published by replica 0. */ |
| 97 | services: FlyMachineService[] | undefined; |
| 98 | count: number; |
| 99 | mounts: MountedDisk[]; |
| 100 | replicas: Replica[]; |
| 101 | } |
| 102 | |
| 103 | const compactRecord = ( |
| 104 | record: Record<string, string | undefined> | null | undefined, |
| 105 | ): Record<string, string> => |
| 106 | Object.fromEntries( |
| 107 | Object.entries(record ?? {}).flatMap(([key, value]) => |
| 108 | value === undefined ? [] : [[key, value]], |
| 109 | ), |
| 110 | ); |
| 111 | |
| 112 | export const gone = (machine: FlyMachine | undefined) => |
| 113 | machine === undefined || machine.state === "destroyed"; |
| 114 | |
| 115 | export const getMachineById = (appName: string, machineId: string) => |
| 116 | machines.getMachine({ app_name: appName, machine_id: machineId }).pipe( |
| 117 | Effect.map((machine) => (gone(machine) ? undefined : machine)), |
| 118 | Effect.catchTag("NotFound", () => Effect.undefined), |
| 119 | ); |
| 120 | |
| 121 | /** |
| 122 | * Reads a Machine we hold a lease on. Fly's Machines API is not |
| 123 | * read-after-write consistent: a freshly created Machine can accept a lease |
| 124 | * (201) and then answer `GET /machines/{id}` with 404 for several seconds. |
| 125 | * Holding the lease blocks other writers, so a 404 here is lag rather than |
| 126 | * deletion; retry it (bounded) before surfacing `NotFound`. |
| 127 | */ |
| 128 | const getLeasedMachine = (appName: string, machineId: string) => |
| 129 | machines.getMachine({ app_name: appName, machine_id: machineId }).pipe( |
| 130 | Effect.retry({ |
| 131 | times: 10, |
| 132 | schedule: Schedule.spaced("1 second"), |
| 133 | while: (error) => error._tag === "NotFound", |
| 134 | }), |
| 135 | ); |
| 136 | |
| 137 | const getLeasedMachineById = (appName: string, machineId: string) => |
| 138 | getLeasedMachine(appName, machineId).pipe( |
| 139 | Effect.map((machine) => (gone(machine) ? undefined : machine)), |
| 140 | Effect.catchTag("NotFound", () => Effect.undefined), |
| 141 | ); |
| 142 | |
| 143 | export const listMachinesByApp = (appName: string) => |
| 144 | machines.listMachines({ app_name: appName }).pipe( |
| 145 | // A just-created App can answer 500 "failed to get app: sql: no rows in |
| 146 | // result set" until the Machines API sees it; safe to replay a read. |
| 147 | Effect.retry({ |
| 148 | while: (error) => error._tag === "InternalServerError", |
| 149 | schedule: Schedule.spaced("2 seconds"), |
| 150 | times: 8, |
| 151 | }), |
| 152 | Effect.map((machines) => machines.filter((machine) => !gone(machine))), |
| 153 | Effect.catchTag("NotFound", () => Effect.succeed([])), |
| 154 | ); |
| 155 | |
| 156 | export const resolveCount = (count: number | undefined) => Math.max(1, Math.floor(count ?? 1)); |
| 157 | |
| 158 | export const replicaMachineName = (base: string, index: number, count: number) => { |
| 159 | if (count <= 1 && index === 0) return base; |
| 160 | const suffix = `-${index}`; |
| 161 | const room = 30 - suffix.length; |
| 162 | const clipped = base.slice(0, Math.max(1, room)).replace(/-+$/g, ""); |
| 163 | return sanitizeFlyAppName(`${clipped}${suffix}`); |
| 164 | }; |
| 165 | |
| 166 | export const replicaIndexOf = (machine: FlyMachine): number => { |
| 167 | const raw = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.replica]; |
| 168 | const parsed = raw === undefined ? 0 : Number(raw); |
| 169 | return Number.isFinite(parsed) ? parsed : 0; |
| 170 | }; |
| 171 | |
| 172 | export const alchemyIdOf = (machine: FlyMachine): string | undefined => { |
| 173 | const id = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.id]; |
| 174 | return id !== undefined && id.length > 0 ? id : undefined; |
| 175 | }; |
| 176 | |
| 177 | export const isOwnedType = (machine: FlyMachine, type: FlyAlchemyType) => { |
| 178 | const metadata = compactRecord(machine.config?.metadata); |
| 179 | return isAlchemyOwnedMetadata(metadata) && metadata[alchemyMetadataKeys.type] === type; |
| 180 | }; |
| 181 | |
| 182 | export const toImageRef = (ref: FlyImageRef | undefined): MachineImageRef | undefined => { |
| 183 | if (ref === undefined) return undefined; |
| 184 | const imageRef: MachineImageRef = { |
| 185 | registry: ref.registry, |
| 186 | repository: ref.repository, |
| 187 | tag: ref.tag, |
| 188 | digest: ref.digest, |
| 189 | }; |
| 190 | return imageRef.registry === undefined && |
| 191 | imageRef.repository === undefined && |
| 192 | imageRef.tag === undefined && |
| 193 | imageRef.digest === undefined |
| 194 | ? undefined |
| 195 | : imageRef; |
| 196 | }; |
| 197 | |
| 198 | export const toGuestAttrs = ( |
| 199 | guest: |
| 200 | | { cpu_kind?: string; cpus?: number; memory_mb?: number; gpu_kind?: string; gpus?: number } |
| 201 | | undefined, |
| 202 | ): MachineGuest | undefined => { |
| 203 | if (guest === undefined) return undefined; |
| 204 | return { |
| 205 | cpuKind: guest.cpu_kind, |
| 206 | cpus: guest.cpus, |
| 207 | memoryMb: guest.memory_mb, |
| 208 | gpuKind: guest.gpu_kind, |
| 209 | gpus: guest.gpus, |
| 210 | }; |
| 211 | }; |
| 212 | |
| 213 | export const toFlyServiceCheck = (check: MachineServiceCheck): FlyMachineServiceCheck => ({ |
| 214 | type: check.type, |
| 215 | port: check.port, |
| 216 | interval: check.interval, |
| 217 | timeout: check.timeout, |
| 218 | grace_period: check.gracePeriod, |
| 219 | method: check.method, |
| 220 | path: check.path, |
| 221 | protocol: check.protocol, |
| 222 | headers: check.headers?.map((header) => ({ name: header.name, values: header.values })), |
| 223 | tls_server_name: check.tlsServerName, |
| 224 | tls_skip_verify: check.tlsSkipVerify, |
| 225 | }); |
| 226 | |
| 227 | export const toFlyService = (service: MachineService): FlyMachineService => ({ |
| 228 | protocol: service.protocol, |
| 229 | internal_port: service.internalPort, |
| 230 | autostart: service.autostart, |
| 231 | autostop: |
| 232 | typeof service.autostop === "boolean" ? (service.autostop ? "stop" : "off") : service.autostop, |
| 233 | min_machines_running: service.minMachinesRunning, |
| 234 | ports: service.ports?.map((port) => ({ |
| 235 | port: port.port, |
| 236 | handlers: port.handlers, |
| 237 | force_https: port.forceHttps, |
| 238 | start_port: port.startPort, |
| 239 | end_port: port.endPort, |
| 240 | })), |
| 241 | checks: service.checks?.map((check) => ({ |
| 242 | ...toFlyServiceCheck(check), |
| 243 | // Fly clamps service-check intervals; named Machine checks retain longer intervals. |
| 244 | interval: |
| 245 | (durationNanoseconds(check.interval) ?? 0n) > 60_000_000_000n ? "60s" : check.interval, |
| 246 | })), |
| 247 | }); |
| 248 | |
| 249 | export const autostopMode = (value: string | boolean | undefined) => |
| 250 | value === true ? "stop" : value === false || value === undefined ? "off" : value; |
| 251 | |
| 252 | const normalizedCheckDuration = (value: string | undefined) => { |
| 253 | const nanos = durationNanoseconds(value); |
| 254 | return nanos === undefined ? value : `${nanos}ns`; |
| 255 | }; |
| 256 | |
| 257 | const normalizedCheck = (check: machines.FlyMachineCheck | FlyMachineServiceCheck | undefined) => |
| 258 | check === undefined |
| 259 | ? undefined |
| 260 | : { |
| 261 | ...check, |
| 262 | grace_period: normalizedCheckDuration(check.grace_period), |
| 263 | interval: normalizedCheckDuration(check.interval), |
| 264 | timeout: normalizedCheckDuration(check.timeout), |
| 265 | }; |
| 266 | |
| 267 | export const sameChecks = ( |
| 268 | observed: FlyMachineConfig["checks"], |
| 269 | desired: FlyMachineConfig["checks"], |
| 270 | ) => { |
| 271 | const normalize = (checks: FlyMachineConfig["checks"]) => |
| 272 | Object.fromEntries( |
| 273 | Object.entries(checks ?? {}).map(([name, check]) => [name, normalizedCheck(check)]), |
| 274 | ); |
| 275 | return deepEqual(normalize(observed), normalize(desired), { stripNullish: true }); |
| 276 | }; |
| 277 | |
| 278 | export const normalizedServices = (services: FlyMachineService[] | undefined) => |
| 279 | (services ?? []).map((service) => ({ |
| 280 | ...service, |
| 281 | autostop: autostopMode(service.autostop), |
| 282 | checks: service.checks?.map(normalizedCheck), |
| 283 | })); |
| 284 | |
| 285 | export const sameServices = ( |
| 286 | observed: FlyMachineService[] | undefined, |
| 287 | desired: FlyMachineService[] | undefined, |
| 288 | ) => deepEqual(normalizedServices(observed), normalizedServices(desired), { stripNullish: true }); |
| 289 | |
| 290 | export const hasPublishedService = (services: FlyMachineService[] | undefined) => |
| 291 | (services ?? []).some((service) => |
| 292 | (service.ports ?? []).some((port) => port.port !== undefined || port.start_port !== undefined), |
| 293 | ); |
| 294 | |
| 295 | export const waitStarted = (appName: string, machineId: string) => |
| 296 | machines |
| 297 | .waitMachine({ |
| 298 | app_name: appName, |
| 299 | machine_id: machineId, |
| 300 | state: "started", |
| 301 | timeout: WAIT_TIMEOUT_SECONDS, |
| 302 | }) |
| 303 | .pipe( |
| 304 | Retry.none, |
| 305 | Effect.retry({ |
| 306 | times: 6, |
| 307 | schedule: waitBackoff, |
| 308 | while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout", |
| 309 | }), |
| 310 | Effect.timeout("50 seconds"), |
| 311 | ); |
| 312 | |
| 313 | export const configuredCheckNames = (config: FlyMachineConfig | undefined) => [ |
| 314 | ...Object.keys(config?.checks ?? {}), |
| 315 | ...(config?.services ?? []).flatMap((service) => |
| 316 | (service.checks ?? []).map( |
| 317 | (check, checkIndex) => |
| 318 | `${SERVICE_CHECK_NAME_PREFIX}${String(checkIndex).padStart(2, "0")}-${check.type ?? "tcp"}-${service.internal_port}`, |
| 319 | ), |
| 320 | ), |
| 321 | ]; |
| 322 | |
| 323 | export const checksPassing = (machine: FlyMachine, config: FlyMachineConfig | undefined) => { |
| 324 | const expected = configuredCheckNames(config); |
| 325 | const checks = machine.checks ?? []; |
| 326 | // Fly may also report compatibility mirrors; they do not replace service reports. |
| 327 | return ( |
| 328 | machine.state === "started" && |
| 329 | new Set(expected).size === expected.length && |
| 330 | checks.every( |
| 331 | (check) => |
| 332 | check.name !== undefined && |
| 333 | check.status === "passing" && |
| 334 | (expected.includes(check.name) || |
| 335 | expected.some( |
| 336 | (name) => |
| 337 | name.startsWith(SERVICE_CHECK_NAME_PREFIX) && |
| 338 | check.name === name.replace(SERVICE_CHECK_NAME_PREFIX, "bg_deployments_compat-"), |
| 339 | )), |
| 340 | ) && |
| 341 | new Set(checks.map((check) => check.name)).size === checks.length && |
| 342 | expected.every((name) => { |
| 343 | const reports = checks.filter((check) => check.name === name); |
| 344 | return reports.length === 1 && reports[0]?.status === "passing"; |
| 345 | }) |
| 346 | ); |
| 347 | }; |
| 348 | |
| 349 | const TRANSIENT_GET_TAGS = [ |
| 350 | "TooManyRequests", |
| 351 | "InternalServerError", |
| 352 | "BadGateway", |
| 353 | "ServiceUnavailable", |
| 354 | "GatewayTimeout", |
| 355 | ] as const; |
| 356 | |
| 357 | /** |
| 358 | * After the Machine is started, wait until Fly reports every service |
| 359 | * check as passing. No configured checks is a no-op. Empty live |
| 360 | * `servicecheck-*` results keep polling — they are not success. |
| 361 | * `warning` / `unknown` during grace keep polling. |
| 362 | */ |
| 363 | export const waitHealthy = Effect.fn(function* ( |
| 364 | appName: string, |
| 365 | machine: FlyMachine, |
| 366 | healthTimeoutMs = 60_000, |
| 367 | config: FlyMachineConfig | undefined = machine.config, |
| 368 | ) { |
| 369 | const machineId = machine.id; |
| 370 | const named = Object.keys(config?.checks ?? {}); |
| 371 | const expected = (config?.services ?? []).flatMap((service) => service.checks ?? []).length; |
| 372 | if (machineId === undefined || (expected === 0 && named.length === 0)) return machine; |
| 373 | |
| 374 | let observed = machine; |
| 375 | const notPassing = () => |
| 376 | new ReplicaChecksNotPassing({ |
| 377 | appName, |
| 378 | machineId, |
| 379 | checks: (observed.checks ?? []).map((check) => ({ |
| 380 | name: check.name, |
| 381 | status: check.status, |
| 382 | output: undefined, |
| 383 | })), |
| 384 | }); |
| 385 | const passing = yield* getMachineById(appName, machineId).pipe( |
| 386 | Retry.none, |
| 387 | Effect.map((current) => { |
| 388 | if (current === undefined) return false; |
| 389 | observed = current; |
| 390 | return ( |
| 391 | current.instance_id !== undefined && |
| 392 | current.instance_id === machine.instance_id && |
| 393 | checksPassing(current, config) |
| 394 | ); |
| 395 | }), |
| 396 | Effect.catchTag(TRANSIENT_GET_TAGS, () => Effect.succeed(false)), |
| 397 | Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)), |
| 398 | Effect.repeat({ |
| 399 | schedule: Schedule.spaced(healthTimeoutMs / 10), |
| 400 | until: (passing) => passing, |
| 401 | times: 10, |
| 402 | }), |
| 403 | Effect.timeoutOrElse({ duration: healthTimeoutMs, orElse: () => Effect.fail(notPassing()) }), |
| 404 | ); |
| 405 | if (!passing) return yield* notPassing(); |
| 406 | return observed; |
| 407 | }); |
| 408 | |
| 409 | export const waitDestroyed = (appName: string, machineId: string) => |
| 410 | machines |
| 411 | .waitMachine({ |
| 412 | app_name: appName, |
| 413 | machine_id: machineId, |
| 414 | state: "destroyed", |
| 415 | timeout: WAIT_TIMEOUT_SECONDS, |
| 416 | }) |
| 417 | .pipe( |
| 418 | Retry.none, |
| 419 | Effect.as(undefined), |
| 420 | Effect.catchTag("NotFound", () => Effect.void), |
| 421 | Effect.retry({ |
| 422 | times: 6, |
| 423 | schedule: waitBackoff, |
| 424 | while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout", |
| 425 | }), |
| 426 | Effect.timeout("50 seconds"), |
| 427 | ); |
| 428 | |
| 429 | export const ensureStarted = ( |
| 430 | appName: string, |
| 431 | machine: FlyMachine, |
| 432 | skipLaunch: boolean, |
| 433 | healthTimeoutMs = 60_000, |
| 434 | expectedConfig?: FlyMachineConfig, |
| 435 | existingLeases?: MachineLeases, |
| 436 | ) => |
| 437 | usingMachineLeases(appName, existingLeases, (leases) => |
| 438 | ensureLeasedStarted(appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases), |
| 439 | ); |
| 440 | |
| 441 | const ensureLeasedStarted = Effect.fn(function* ( |
| 442 | appName: string, |
| 443 | machine: FlyMachine, |
| 444 | skipLaunch: boolean, |
| 445 | healthTimeoutMs: number, |
| 446 | expectedConfig: FlyMachineConfig | undefined, |
| 447 | leases: MachineLeases, |
| 448 | ) { |
| 449 | const machineId = machine.id; |
| 450 | if (machineId === undefined || skipLaunch) return machine; |
| 451 | yield* leases.acquire([machineId]); |
| 452 | const started = yield* Effect.gen(function* () { |
| 453 | // Create/update responses can lag Fly's automatic launch. |
| 454 | const current = yield* getLeasedMachine(appName, machineId); |
| 455 | if (!sameOwnership(current, machine)) { |
| 456 | return yield* new ReplicaOwnershipChanged({ appName, machineId }); |
| 457 | } |
| 458 | yield* Effect.logDebug("Fly machine startup", { appName, machineId, state: current.state }); |
| 459 | if ( |
| 460 | current.state === "stopped" || |
| 461 | current.state === "suspended" || |
| 462 | current.state === "failed" |
| 463 | ) { |
| 464 | yield* leases.mutate(machineId, (lease_nonce) => |
| 465 | machines.startMachine({ app_name: appName, machine_id: machineId, lease_nonce }), |
| 466 | ); |
| 467 | } |
| 468 | // Re-observe state between waits instead of retrying the wait in the SDK. |
| 469 | yield* machines |
| 470 | .waitMachine({ |
| 471 | app_name: appName, |
| 472 | machine_id: machineId, |
| 473 | state: "started", |
| 474 | instance_id: current.instance_id, |
| 475 | timeout: WAIT_TIMEOUT_SECONDS, |
| 476 | }) |
| 477 | .pipe(Retry.none); |
| 478 | return yield* getLeasedMachine(appName, machineId); |
| 479 | }).pipe( |
| 480 | Effect.retry({ |
| 481 | times: 6, |
| 482 | schedule: waitBackoff, |
| 483 | while: (error) => |
| 484 | error._tag === "MachineStartFromCreatedState" || |
| 485 | error._tag === "MachineReplacing" || |
| 486 | error._tag === "MachineWaitTimeout" || |
| 487 | error._tag === "Conflict" || |
| 488 | error._tag === "GatewayTimeout", |
| 489 | }), |
| 490 | Effect.timeout("180 seconds"), |
| 491 | ); |
| 492 | return yield* waitHealthy(appName, started, healthTimeoutMs, expectedConfig ?? machine.config); |
| 493 | }); |
| 494 | |
| 495 | export const deleteMachine = (appName: string, machineId: string, existingLeases?: MachineLeases) => |
| 496 | usingMachineLeases(appName, existingLeases, (leases) => |
| 497 | Effect.gen(function* () { |
| 498 | if (appName.length === 0 || machineId.length === 0) return; |
| 499 | yield* leases.acquire([machineId]); |
| 500 | yield* leases.remove(machineId, (lease_nonce) => |
| 501 | machines |
| 502 | .deleteMachine({ app_name: appName, machine_id: machineId, force: true, lease_nonce }) |
| 503 | .pipe(Effect.timeout("30 seconds")), |
| 504 | ); |
| 505 | yield* waitDestroyed(appName, machineId); |
| 506 | }).pipe(Effect.catchTag("NotFound", () => leases.forget(machineId))), |
| 507 | ); |
| 508 | |
| 509 | const durationNanoseconds = (value: string | undefined) => { |
| 510 | if (value === undefined || value === "") return undefined; |
| 511 | const negative = value.startsWith("-"); |
| 512 | const duration = /^[+-]/.test(value) ? value.slice(1) : value; |
| 513 | if (duration === "0") return 0n; |
| 514 | const limit = negative ? 1n << 63n : (1n << 63n) - 1n; |
| 515 | const units: Record<string, bigint> = { |
| 516 | ns: 1n, |
| 517 | us: 1_000n, |
| 518 | µs: 1_000n, |
| 519 | μs: 1_000n, |
| 520 | ms: 1_000_000n, |
| 521 | s: 1_000_000_000n, |
| 522 | m: 60_000_000_000n, |
| 523 | h: 3_600_000_000_000n, |
| 524 | }; |
| 525 | let total = 0n; |
| 526 | let consumed = 0; |
| 527 | for (const part of duration.matchAll(/(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gy)) { |
| 528 | const [integer = "", fraction = ""] = part[1]!.split("."); |
| 529 | const whole = integer.replace(/^0+/, "") || "0"; |
| 530 | if (whole.length > 19) return undefined; |
| 531 | const unit = units[part[2]!]!; |
| 532 | let fractionalNanos = 0n; |
| 533 | // Truncate each component to nanoseconds without unbounded BigInt operands. |
| 534 | for (let index = fraction.length - 1; index >= 0; index--) { |
| 535 | fractionalNanos = (BigInt(fraction[index]!) * unit + fractionalNanos) / 10n; |
| 536 | } |
| 537 | total += BigInt(whole) * unit + fractionalNanos; |
| 538 | if (total > limit) return undefined; |
| 539 | consumed += part[0].length; |
| 540 | } |
| 541 | if (consumed === 0 || consumed !== duration.length) return undefined; |
| 542 | return negative ? -total : total; |
| 543 | }; |
| 544 | |
| 545 | const stopTimeoutMillis = (timeout: string | undefined) => { |
| 546 | if (timeout === "") return 0; |
| 547 | // Shutdown policies retain their unsigned millisecond-or-larger syntax. |
| 548 | if (timeout === undefined || !/^(?:\d+(?:\.\d+)?(?:ms|s|m|h))+$/.test(timeout)) { |
| 549 | return undefined; |
| 550 | } |
| 551 | const nanos = durationNanoseconds(timeout); |
| 552 | return nanos === undefined ? undefined : Number(nanos) / 1_000_000; |
| 553 | }; |
| 554 | |
| 555 | export const sameStopConfig = ( |
| 556 | observed: FlyStopConfig | undefined, |
| 557 | desired: FlyStopConfig | undefined, |
| 558 | ) => |
| 559 | observed?.signal === desired?.signal && |
| 560 | stopTimeoutMillis(observed?.timeout) === stopTimeoutMillis(desired?.timeout); |
| 561 | |
| 562 | export class ShutdownPolicyMismatch extends Data.TaggedError("Fly.ShutdownPolicyMismatch")<{ |
| 563 | machineId: string; |
| 564 | message: string; |
| 565 | }> {} |
| 566 | |
| 567 | export const predecessorShutdown = Effect.fn(function* (machine: FlyMachine) { |
| 568 | const persisted = machine.config?.stop_config; |
| 569 | const injected = machine.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS; |
| 570 | const timeoutMs = stopTimeoutMillis(persisted?.timeout); |
| 571 | if (injected !== undefined) { |
| 572 | const managedTimeout = Number(injected); |
| 573 | const signal = persisted?.signal ?? "SIGTERM"; |
| 574 | if ( |
| 575 | !/^\d+$/.test(injected) || |
| 576 | !Number.isSafeInteger(managedTimeout) || |
| 577 | managedTimeout <= 0 || |
| 578 | managedTimeout > 300_000 || |
| 579 | (signal !== "SIGTERM" && signal !== "SIGINT") || |
| 580 | (persisted?.timeout !== undefined && timeoutMs !== managedTimeout) |
| 581 | ) { |
| 582 | return yield* new ShutdownPolicyMismatch({ |
| 583 | machineId: machine.id ?? "", |
| 584 | message: |
| 585 | "The predecessor's managed shutdown environment and stop_config disagree; reconcile its shutdown policy before migration.", |
| 586 | }); |
| 587 | } |
| 588 | return { |
| 589 | signal, |
| 590 | timeout: persisted?.timeout ?? `${managedTimeout}ms`, |
| 591 | timeoutMs: managedTimeout, |
| 592 | }; |
| 593 | } |
| 594 | if (persisted?.timeout !== undefined && (!timeoutMs || timeoutMs > 300_000)) { |
| 595 | return yield* new ShutdownPolicyMismatch({ |
| 596 | machineId: machine.id ?? "", |
| 597 | message: |
| 598 | "The predecessor has an invalid stop_config timeout; refusing to shorten its shutdown window.", |
| 599 | }); |
| 600 | } |
| 601 | // Unspecified raw-image overrides must remain unspecified for Fly's defaults. |
| 602 | return { |
| 603 | signal: persisted?.signal, |
| 604 | timeout: persisted?.timeout, |
| 605 | timeoutMs: timeoutMs ?? 300_000, |
| 606 | }; |
| 607 | }); |
| 608 | |
| 609 | export class ReplicaOwnershipChanged extends Data.TaggedError("Fly.ReplicaOwnershipChanged")<{ |
| 610 | appName: string; |
| 611 | machineId: string; |
| 612 | }> {} |
| 613 | |
| 614 | export class ReplicaRetirementIncomplete extends Data.TaggedError( |
| 615 | "Fly.ReplicaRetirementIncomplete", |
| 616 | )<{ appName: string; residuals: Array<{ machineId: string; stage: string }> }> {} |
| 617 | |
| 618 | const retirementStep = |
| 619 | (appName: string, machineId: string, stage: string) => |
| 620 | <A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) => |
| 621 | effect.pipe( |
| 622 | Effect.mapError((error) => |
| 623 | error._tag === "NotFound" |
| 624 | ? error |
| 625 | : new ReplicaRetirementIncomplete({ |
| 626 | appName, |
| 627 | residuals: [{ machineId, stage: `${stage}: ${error._tag}` }], |
| 628 | }), |
| 629 | ), |
| 630 | ); |
| 631 | |
| 632 | const sameOwnership = (observed: FlyMachine, snapshot: FlyMachine) => |
| 633 | observed.id === snapshot.id && |
| 634 | [ |
| 635 | alchemyMetadataKeys.stack, |
| 636 | alchemyMetadataKeys.stage, |
| 637 | alchemyMetadataKeys.id, |
| 638 | alchemyMetadataKeys.type, |
| 639 | alchemyMetadataKeys.instance, |
| 640 | alchemyMetadataKeys.fqn, |
| 641 | alchemyMetadataKeys.generation, |
| 642 | ].every((key) => observed.config?.metadata?.[key] === snapshot.config?.metadata?.[key]); |
| 643 | |
| 644 | export const leaseSnapshot = Effect.fn(function* ( |
| 645 | appName: string, |
| 646 | snapshot: FlyMachine[], |
| 647 | leases: MachineLeases, |
| 648 | ) { |
| 649 | const ordered = [...snapshot].sort((a, b) => (a.id ?? "").localeCompare(b.id ?? "")); |
| 650 | const leased = new Set<string>(); |
| 651 | yield* Effect.forEach( |
| 652 | ordered, |
| 653 | (machine) => |
| 654 | Effect.gen(function* () { |
| 655 | if (!machine.id || machine.host_status === "unreachable") return; |
| 656 | const id = machine.id; |
| 657 | yield* leases.acquire([id]).pipe( |
| 658 | Effect.tap(() => Effect.sync(() => leased.add(id))), |
| 659 | // A just-created Machine can 404 on the lease API while it already |
| 660 | // lists; a Machine that is really gone is dropped by the GET below. |
| 661 | Effect.retry({ |
| 662 | while: (error) => error._tag === "NotFound", |
| 663 | schedule: Schedule.spaced("1 second"), |
| 664 | times: 5, |
| 665 | }), |
| 666 | Effect.catchTag("NotFound", () => Effect.void), |
| 667 | ); |
| 668 | }), |
| 669 | { concurrency: 1 }, |
| 670 | ).pipe(Effect.timeout("30 seconds")); |
| 671 | const observed: FlyMachine[] = []; |
| 672 | for (const machine of ordered) { |
| 673 | if (!machine.id) continue; |
| 674 | if (machine.host_status === "unreachable") { |
| 675 | observed.push(machine); |
| 676 | continue; |
| 677 | } |
| 678 | const current = yield* leased.has(machine.id) |
| 679 | ? getLeasedMachineById(appName, machine.id) |
| 680 | : getMachineById(appName, machine.id); |
| 681 | if (!current) { |
| 682 | yield* leases.forget(machine.id); |
| 683 | continue; |
| 684 | } |
| 685 | if (!sameOwnership(current, machine)) |
| 686 | return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id }); |
| 687 | yield* leases.checkTarget(machine.id); |
| 688 | observed.push(current); |
| 689 | } |
| 690 | yield* leases.check; |
| 691 | return observed; |
| 692 | }); |
| 693 | |
| 694 | export const retireMachines = ( |
| 695 | appName: string, |
| 696 | snapshot: FlyMachine[], |
| 697 | replacementReady = false, |
| 698 | existingLeases?: MachineLeases, |
| 699 | ) => |
| 700 | usingMachineLeases(appName, existingLeases, (leases) => |
| 701 | retireLeasedMachines(appName, snapshot, replacementReady, leases), |
| 702 | ); |
| 703 | |
| 704 | const retireLeasedMachines = Effect.fn(function* ( |
| 705 | appName: string, |
| 706 | snapshot: FlyMachine[], |
| 707 | replacementReady: boolean, |
| 708 | leases: MachineLeases, |
| 709 | ) { |
| 710 | const targets = yield* leaseSnapshot(appName, snapshot, leases); |
| 711 | const results = yield* Effect.forEach( |
| 712 | targets, |
| 713 | (machine) => |
| 714 | Effect.gen(function* () { |
| 715 | yield* leases.check; |
| 716 | const result = yield* Effect.gen(function* () { |
| 717 | if (!machine.id) return; |
| 718 | if (replacementReady && machine.host_status === "unreachable") { |
| 719 | const metadata = machine.config?.metadata; |
| 720 | if ( |
| 721 | !machine.config || |
| 722 | machine.incomplete_config || |
| 723 | (machine.config.mounts?.length ?? 0) > 0 || |
| 724 | !metadata?.[alchemyMetadataKeys.instance] || |
| 725 | !metadata[alchemyMetadataKeys.fqn] |
| 726 | ) { |
| 727 | return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id }); |
| 728 | } |
| 729 | yield* Effect.logWarning( |
| 730 | "Force-retiring owned stateless Fly Machine on unreachable host; work was not proven drained", |
| 731 | { appName, machineId: machine.id }, |
| 732 | ); |
| 733 | const machineId = machine.id; |
| 734 | const heldNonce = yield* leases.nonceIfHeld(machineId); |
| 735 | yield* Effect.gen(function* () { |
| 736 | const remove = (lease_nonce?: string) => |
| 737 | machines |
| 738 | .deleteMachine({ |
| 739 | app_name: appName, |
| 740 | machine_id: machineId, |
| 741 | force: true, |
| 742 | lease_nonce, |
| 743 | }) |
| 744 | .pipe(Retry.none, Effect.timeout("30 seconds")); |
| 745 | if (heldNonce !== undefined) yield* leases.remove(machineId, remove); |
| 746 | else yield* leases.guard(remove()); |
| 747 | }).pipe( |
| 748 | Effect.catchTag("NotFound", () => Effect.void), |
| 749 | retirementStep(appName, machineId, "unreachable force-delete"), |
| 750 | ); |
| 751 | yield* leases.forget(machine.id); |
| 752 | return yield* waitDestroyed(appName, machine.id); |
| 753 | } |
| 754 | yield* retireMachine(appName, machine.id, machine, leases); |
| 755 | }).pipe(Effect.result); |
| 756 | yield* leases.check; |
| 757 | return Result.isFailure(result) |
| 758 | ? result.failure._tag === "Fly.ReplicaRetirementIncomplete" |
| 759 | ? result.failure.residuals |
| 760 | : [{ machineId: machine.id ?? "", stage: result.failure._tag }] |
| 761 | : []; |
| 762 | }), |
| 763 | { concurrency: 4 }, |
| 764 | ); |
| 765 | const residuals = results.flat(); |
| 766 | if (residuals.length) return yield* new ReplicaRetirementIncomplete({ appName, residuals }); |
| 767 | }); |
| 768 | |
| 769 | export const retireMachine = ( |
| 770 | appName: string, |
| 771 | machineId: string, |
| 772 | snapshot?: FlyMachine, |
| 773 | existingLeases?: MachineLeases, |
| 774 | ) => |
| 775 | usingMachineLeases(appName, existingLeases, (leases) => |
| 776 | retireLeasedMachine(appName, machineId, snapshot, leases).pipe( |
| 777 | Effect.tap(() => leases.forget(machineId)), |
| 778 | ), |
| 779 | ); |
| 780 | |
| 781 | const retireLeasedMachine = Effect.fn( |
| 782 | function* ( |
| 783 | appName: string, |
| 784 | machineId: string, |
| 785 | snapshot: FlyMachine | undefined, |
| 786 | leases: MachineLeases, |
| 787 | ) { |
| 788 | yield* leases.acquire([machineId]); |
| 789 | const current = yield* getMachineById(appName, machineId); |
| 790 | if (current === undefined) return; |
| 791 | if (snapshot && !sameOwnership(current, snapshot)) { |
| 792 | return yield* new ReplicaOwnershipChanged({ appName, machineId }); |
| 793 | } |
| 794 | const stop = yield* predecessorShutdown(current); |
| 795 | if ( |
| 796 | current.cordoned === true && |
| 797 | current.config?.metadata?.[alchemyMetadataKeys.phase] === "candidate" |
| 798 | ) |
| 799 | return yield* deleteMachine(appName, machineId, leases); |
| 800 | if (current.state !== "stopped" && current.state !== "created") { |
| 801 | yield* leases |
| 802 | .mutate( |
| 803 | machineId, |
| 804 | (lease_nonce) => |
| 805 | machines |
| 806 | .cordonMachine({ app_name: appName, machine_id: machineId, lease_nonce }) |
| 807 | .pipe(Effect.timeout("30 seconds")), |
| 808 | { idempotent: true }, |
| 809 | ) |
| 810 | .pipe(retirementStep(appName, machineId, "cordon")); |
| 811 | if (hasPublishedService(current.config?.services)) yield* Effect.sleep("10 seconds"); |
| 812 | yield* leases |
| 813 | .mutate( |
| 814 | machineId, |
| 815 | (lease_nonce) => |
| 816 | machines |
| 817 | .stopMachine({ |
| 818 | app_name: appName, |
| 819 | machine_id: machineId, |
| 820 | lease_nonce, |
| 821 | signal: stop.signal, |
| 822 | timeout: stop.timeout, |
| 823 | }) |
| 824 | .pipe(Effect.timeout(stop.timeoutMs + 15_000)), |
| 825 | { timeoutMs: stop.timeoutMs + 120_000 }, |
| 826 | ) |
| 827 | .pipe(retirementStep(appName, machineId, "stop")); |
| 828 | yield* machines |
| 829 | .waitMachine({ |
| 830 | app_name: appName, |
| 831 | machine_id: machineId, |
| 832 | state: "stopped", |
| 833 | instance_id: current.instance_id, |
| 834 | timeout: WAIT_TIMEOUT_SECONDS, |
| 835 | }) |
| 836 | .pipe( |
| 837 | Retry.none, |
| 838 | Effect.retry({ |
| 839 | times: 10, |
| 840 | schedule: Schedule.spaced(Math.max(500, stop.timeoutMs / 10)), |
| 841 | while: (error) => |
| 842 | error._tag === "MachineWaitTimeout" || error._tag === "GatewayTimeout", |
| 843 | }), |
| 844 | Effect.timeout(stop.timeoutMs + 15_000), |
| 845 | retirementStep(appName, machineId, "wait-stopped"), |
| 846 | ); |
| 847 | } |
| 848 | yield* leases |
| 849 | .remove(machineId, (lease_nonce) => |
| 850 | machines |
| 851 | .deleteMachine({ app_name: appName, machine_id: machineId, force: false, lease_nonce }) |
| 852 | .pipe(Effect.timeout("30 seconds")), |
| 853 | ) |
| 854 | .pipe( |
| 855 | Effect.catchTag("NotFound", () => Effect.void), |
| 856 | retirementStep(appName, machineId, "delete"), |
| 857 | ); |
| 858 | yield* waitDestroyed(appName, machineId).pipe( |
| 859 | retirementStep(appName, machineId, "verify-absent"), |
| 860 | ); |
| 861 | }, |
| 862 | Effect.catchTag("NotFound", () => Effect.void), |
| 863 | ); |
| 864 | |
| 865 | const mountedDisksOf = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): MountedDisk[] => |
| 866 | (machine.config?.mounts ?? []).flatMap((mount) => { |
| 867 | const volumeId = mount.volume; |
| 868 | const path = mount.path; |
| 869 | if (volumeId === undefined || path === undefined) return []; |
| 870 | const volume = volumesById.get(volumeId); |
| 871 | return [{ path, volumeId, sizeGb: volume?.size_gb ?? 0, name: volume?.name ?? "" }]; |
| 872 | }); |
| 873 | |
| 874 | export const toReplica = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): Replica => ({ |
| 875 | machineId: machine.id ?? "", |
| 876 | name: machine.name ?? "", |
| 877 | region: machine.region ?? "", |
| 878 | state: machine.state ?? "", |
| 879 | instanceId: machine.instance_id, |
| 880 | privateIp: machine.private_ip, |
| 881 | imageRef: toImageRef(machine.image_ref), |
| 882 | guest: toGuestAttrs(machine.config?.guest), |
| 883 | mounts: mountedDisksOf(machine, volumesById), |
| 884 | }); |
| 885 | |
| 886 | export const toReplicaSet = ( |
| 887 | replicas: Replica[], |
| 888 | appName: string, |
| 889 | baseName: string, |
| 890 | services?: FlyMachineService[], |
| 891 | ): ReplicaSet => { |
| 892 | const primary = replicas[0]; |
| 893 | return { |
| 894 | appName, |
| 895 | machineId: primary?.machineId ?? "", |
| 896 | machineIds: replicas.map((replica) => replica.machineId), |
| 897 | name: primary?.name ?? baseName, |
| 898 | baseName, |
| 899 | region: primary?.region ?? "", |
| 900 | regions: [ |
| 901 | ...new Set(replicas.map((replica) => replica.region).filter((region) => region.length > 0)), |
| 902 | ], |
| 903 | state: primary?.state ?? "", |
| 904 | instanceId: primary?.instanceId, |
| 905 | privateIp: primary?.privateIp, |
| 906 | imageRef: primary?.imageRef, |
| 907 | guest: primary?.guest, |
| 908 | services, |
| 909 | count: replicas.length, |
| 910 | mounts: primary?.mounts ?? [], |
| 911 | replicas, |
| 912 | }; |
| 913 | }; |
| 914 | |
| 915 | export const ownedReplicas = ( |
| 916 | listed: FlyMachine[], |
| 917 | input: { |
| 918 | metadata: Record<string, string>; |
| 919 | resourceInstanceId: string; |
| 920 | fqn: string; |
| 921 | baseName?: string; |
| 922 | machineIds?: readonly string[]; |
| 923 | }, |
| 924 | ) => |
| 925 | listed.filter((machine) => { |
| 926 | const metadata = machine.config?.metadata ?? {}; |
| 927 | if (!Object.entries(input.metadata).every(([key, value]) => metadata[key] === value)) |
| 928 | return false; |
| 929 | if (metadata[alchemyMetadataKeys.instance] !== undefined) { |
| 930 | return ( |
| 931 | metadata[alchemyMetadataKeys.instance] === input.resourceInstanceId && |
| 932 | metadata[alchemyMetadataKeys.fqn] === input.fqn |
| 933 | ); |
| 934 | } |
| 935 | return ( |
| 936 | (machine.id !== undefined && input.machineIds?.includes(machine.id)) || |
| 937 | (input.baseName !== undefined && |
| 938 | (machine.name === input.baseName || |
| 939 | machine.name === replicaMachineName(input.baseName, replicaIndexOf(machine), 2))) |
| 940 | ); |
| 941 | }); |
| 942 | |
| 943 | export const listReplicas = Effect.fn(function* (input: { |
| 944 | appName: string; |
| 945 | id: string; |
| 946 | type: FlyAlchemyType; |
| 947 | }) { |
| 948 | const machines = yield* listMachinesByApp(input.appName); |
| 949 | return machines |
| 950 | .filter((machine) => isOwnedType(machine, input.type) && alchemyIdOf(machine) === input.id) |
| 951 | .sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right)); |
| 952 | }); |
| 953 | |
| 954 | export const listReplicaSets = Effect.fn(function* (type: FlyAlchemyType) { |
| 955 | const apps = yield* listOwnedApps(); |
| 956 | const groups = yield* Effect.forEach( |
| 957 | apps, |
| 958 | (app) => |
| 959 | listMachinesByApp(app.appName).pipe( |
| 960 | Effect.map((machines) => { |
| 961 | const owned = machines.filter((machine) => isOwnedType(machine, type)); |
| 962 | const byId = new Map<string, FlyMachine[]>(); |
| 963 | for (const machine of owned) { |
| 964 | const id = alchemyIdOf(machine); |
| 965 | if (id === undefined) continue; |
| 966 | const metadata = machine.config?.metadata; |
| 967 | const key = JSON.stringify([ |
| 968 | metadata?.[alchemyMetadataKeys.stack], |
| 969 | metadata?.[alchemyMetadataKeys.stage], |
| 970 | metadata?.[alchemyMetadataKeys.fqn] ?? id, |
| 971 | metadata?.[alchemyMetadataKeys.instance], |
| 972 | metadata?.[alchemyMetadataKeys.generation], |
| 973 | ]); |
| 974 | const group = byId.get(key) ?? []; |
| 975 | group.push(machine); |
| 976 | byId.set(key, group); |
| 977 | } |
| 978 | return [...byId.values()].map((group) => { |
| 979 | const sorted = [...group].sort( |
| 980 | (left, right) => replicaIndexOf(left) - replicaIndexOf(right), |
| 981 | ); |
| 982 | const replicas = sorted.map((machine) => toReplica(machine, new Map())); |
| 983 | return toReplicaSet( |
| 984 | replicas, |
| 985 | app.appName, |
| 986 | replicas[0]?.name ?? "", |
| 987 | sorted[0]?.config?.services, |
| 988 | ); |
| 989 | }); |
| 990 | }), |
| 991 | ), |
| 992 | { concurrency: 8 }, |
| 993 | ); |
| 994 | return groups.flat(); |
| 995 | }); |
| 996 | |
| 997 | const pickVolume = ( |
| 998 | group: FlyVolume[], |
| 999 | used: Set<string>, |
| 1000 | preferId: string | undefined, |
| 1001 | ): FlyVolume | undefined => { |
| 1002 | if (preferId !== undefined && !used.has(preferId)) { |
| 1003 | const preferred = group.find((volume) => volume.id === preferId); |
| 1004 | if (preferred !== undefined) return preferred; |
| 1005 | } |
| 1006 | return group.find((volume) => volume.id !== undefined && !used.has(volume.id)); |
| 1007 | }; |
| 1008 | |
| 1009 | export interface ReconcileReplicasInput { |
| 1010 | fqn: string; |
| 1011 | resourceInstanceId: string; |
| 1012 | policy: DeploymentPolicy; |
| 1013 | checks?: Record<string, MachineCheck>; |
| 1014 | id: string; |
| 1015 | type: FlyAlchemyType; |
| 1016 | appName: string; |
| 1017 | baseName: string; |
| 1018 | /** |
| 1019 | * Regions to run in. `count` is the total replica count; replica `i` |
| 1020 | * runs in `regions[i % regions.length]`. |
| 1021 | */ |
| 1022 | regions: readonly string[]; |
| 1023 | count: number; |
| 1024 | disks: DiskSpec[]; |
| 1025 | skipLaunch?: boolean; |
| 1026 | minSecretsVersion?: number; |
| 1027 | outputMachineIds?: readonly string[]; |
| 1028 | preferVolumeIds?: ReadonlyArray<ReadonlyArray<string>>; |
| 1029 | configDrifted: ( |
| 1030 | machine: FlyMachine, |
| 1031 | desired: { mounts: FlyMachineMount[]; metadata: Record<string, string> }, |
| 1032 | ) => boolean; |
| 1033 | buildConfig: (replica: { |
| 1034 | index: number; |
| 1035 | mounts: FlyMachineMount[]; |
| 1036 | metadata: Record<string, string>; |
| 1037 | }) => FlyMachineConfig; |
| 1038 | } |
| 1039 | |
| 1040 | export const reconcileReplicas = (input: ReconcileReplicasInput) => |
| 1041 | usingMachineLeases(input.appName, undefined, (leases) => reconcileLeasedReplicas(input, leases)); |
| 1042 | |
| 1043 | const reconcileLeasedReplicas = Effect.fn(function* ( |
| 1044 | input: ReconcileReplicasInput, |
| 1045 | leases: MachineLeases, |
| 1046 | ) { |
| 1047 | const ownership = yield* createMachineMetadata(input.id, input.type); |
| 1048 | const alchemy = { |
| 1049 | ...ownership, |
| 1050 | [alchemyMetadataKeys.instance]: input.resourceInstanceId, |
| 1051 | [alchemyMetadataKeys.fqn]: input.fqn, |
| 1052 | [alchemyMetadataKeys.baseName]: input.baseName, |
| 1053 | }; |
| 1054 | const buildConfig: ReconcileReplicasInput["buildConfig"] = (replica) => ({ |
| 1055 | ...input.buildConfig(replica), |
| 1056 | checks: |
| 1057 | input.checks === undefined |
| 1058 | ? undefined |
| 1059 | : Object.fromEntries( |
| 1060 | Object.entries(input.checks).map(([name, check]) => [name, toFlyServiceCheck(check)]), |
| 1061 | ), |
| 1062 | stop_config: |
| 1063 | input.policy.shutdown === undefined |
| 1064 | ? undefined |
| 1065 | : { signal: input.policy.shutdown.signal, timeout: input.policy.shutdown.timeout }, |
| 1066 | }); |
| 1067 | const config = buildConfig({ index: 0, mounts: [], metadata: alchemy }); |
| 1068 | yield* validateDeployment(input.policy, config, input.disks.length > 0, input.skipLaunch); |
| 1069 | if (input.policy.bluegreen) |
| 1070 | return yield* reconcileBlueGreen({ ...input, buildConfig }, ownership, alchemy, leases); |
| 1071 | const desiredNames = new Set( |
| 1072 | Array.from({ length: input.count }, (_, index) => |
| 1073 | replicaMachineName(input.baseName, index, input.count), |
| 1074 | ), |
| 1075 | ); |
| 1076 | const listed = yield* listMachinesByApp(input.appName); |
| 1077 | const owned = yield* leaseSnapshot( |
| 1078 | input.appName, |
| 1079 | ownedReplicas(listed, { ...input, metadata: ownership, machineIds: input.outputMachineIds }), |
| 1080 | leases, |
| 1081 | ); |
| 1082 | const byIndex = new Map<number, FlyMachine>(); |
| 1083 | const preferIds = new Set((input.outputMachineIds ?? []).filter((id) => id.length > 0)); |
| 1084 | for (const machine of owned) { |
| 1085 | const id = machine.id; |
| 1086 | if (id !== undefined && preferIds.has(id)) { |
| 1087 | byIndex.set(replicaIndexOf(machine), machine); |
| 1088 | } |
| 1089 | } |
| 1090 | for (const machine of owned) { |
| 1091 | const name = machine.name; |
| 1092 | if ( |
| 1093 | name === undefined || |
| 1094 | (!desiredNames.has(name) && |
| 1095 | machine.config?.metadata?.[alchemyMetadataKeys.instance] !== input.resourceInstanceId) |
| 1096 | ) |
| 1097 | continue; |
| 1098 | const index = replicaIndexOf(machine); |
| 1099 | if (!byIndex.has(index)) byIndex.set(index, machine); |
| 1100 | } |
| 1101 | |
| 1102 | const scaleDown = [...byIndex].filter(([index]) => index >= input.count); |
| 1103 | yield* retireMachines( |
| 1104 | input.appName, |
| 1105 | scaleDown.map(([, machine]) => machine), |
| 1106 | false, |
| 1107 | leases, |
| 1108 | ); |
| 1109 | for (const [index] of scaleDown) byIndex.delete(index); |
| 1110 | |
| 1111 | // Volumes are regional: one group per disk and region, sized to the |
| 1112 | // replicas that run there. |
| 1113 | const replicasIn = (region: string) => |
| 1114 | Array.from({ length: input.count }, (_, index) => index).filter( |
| 1115 | (index) => regionOfReplica(input.regions, index) === region, |
| 1116 | ).length; |
| 1117 | const groups: Array<{ |
| 1118 | disk: DiskSpec; |
| 1119 | name: string; |
| 1120 | byRegion: Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>; |
| 1121 | }> = []; |
| 1122 | for (const [diskIndex, disk] of input.disks.entries()) { |
| 1123 | const name = yield* volumeGroupName(input.id, disk); |
| 1124 | const preferIds = (input.preferVolumeIds ?? []) |
| 1125 | .map((replica) => replica[diskIndex]) |
| 1126 | .filter((id): id is string => id !== undefined && id.length > 0); |
| 1127 | const byRegion = new Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>(); |
| 1128 | for (const region of input.regions) { |
| 1129 | byRegion.set( |
| 1130 | region, |
| 1131 | yield* ensureVolumeGroup({ |
| 1132 | appName: input.appName, |
| 1133 | name, |
| 1134 | region, |
| 1135 | count: replicasIn(region), |
| 1136 | disk, |
| 1137 | preferIds, |
| 1138 | }), |
| 1139 | ); |
| 1140 | } |
| 1141 | groups.push({ disk, name, byRegion }); |
| 1142 | } |
| 1143 | |
| 1144 | const usedVolumeIds = new Set<string>(); |
| 1145 | const live: FlyMachine[] = []; |
| 1146 | for (let index = 0; index < input.count; index++) { |
| 1147 | const name = replicaMachineName(input.baseName, index, input.count); |
| 1148 | const region = regionOfReplica(input.regions, index); |
| 1149 | const metadata = { |
| 1150 | ...alchemy, |
| 1151 | [alchemyMetadataKeys.replica]: String(index), |
| 1152 | ...(input.minSecretsVersion === undefined |
| 1153 | ? {} |
| 1154 | : { [alchemyMetadataKeys.secretsVersion]: String(input.minSecretsVersion) }), |
| 1155 | }; |
| 1156 | const prefer = input.preferVolumeIds?.[index] ?? []; |
| 1157 | const mounts: FlyMachineMount[] = []; |
| 1158 | for (const [diskIndex, group] of groups.entries()) { |
| 1159 | const volume = pickVolume( |
| 1160 | group.byRegion.get(region)?.volumes ?? [], |
| 1161 | usedVolumeIds, |
| 1162 | prefer[diskIndex], |
| 1163 | ); |
| 1164 | const volumeId = volume?.id; |
| 1165 | if (volumeId === undefined) { |
| 1166 | return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName }); |
| 1167 | } |
| 1168 | usedVolumeIds.add(volumeId); |
| 1169 | mounts.push({ volume: volumeId, path: group.disk.path }); |
| 1170 | } |
| 1171 | const config = buildConfig({ index, mounts, metadata }); |
| 1172 | let current = byIndex.get(index); |
| 1173 | // Fly cannot move a Machine between regions. Machine names are unique |
| 1174 | // per App, so retire the old replica before creating it in its region. |
| 1175 | if (current !== undefined && current.region !== region) { |
| 1176 | yield* retireMachines(input.appName, [current], false, leases); |
| 1177 | byIndex.delete(index); |
| 1178 | current = undefined; |
| 1179 | } |
| 1180 | if (current === undefined) { |
| 1181 | const created = yield* machines |
| 1182 | .createMachine({ |
| 1183 | app_name: input.appName, |
| 1184 | name, |
| 1185 | region, |
| 1186 | config, |
| 1187 | skip_launch: input.skipLaunch === true ? true : undefined, |
| 1188 | min_secrets_version: input.minSecretsVersion, |
| 1189 | }) |
| 1190 | .pipe(Effect.catchTag("Conflict", () => Effect.undefined)); |
| 1191 | current = |
| 1192 | created ?? |
| 1193 | (yield* listMachinesByApp(input.appName).pipe( |
| 1194 | Effect.map((machines) => |
| 1195 | ownedReplicas(machines, { |
| 1196 | ...input, |
| 1197 | metadata: ownership, |
| 1198 | machineIds: input.outputMachineIds, |
| 1199 | }).find((machine) => machine.name === name), |
| 1200 | ), |
| 1201 | )); |
| 1202 | if (current === undefined || current.id === undefined) { |
| 1203 | return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName }); |
| 1204 | } |
| 1205 | const observed = (yield* leaseSnapshot(input.appName, [current], leases))[0]; |
| 1206 | if (!observed || input.configDrifted(observed, { mounts, metadata })) { |
| 1207 | return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName }); |
| 1208 | } |
| 1209 | current = observed; |
| 1210 | } else if ( |
| 1211 | !sameChecks(current.config?.checks, config.checks) || |
| 1212 | !sameStopConfig(current.config?.stop_config, config.stop_config) || |
| 1213 | input.configDrifted(current, { mounts, metadata }) |
| 1214 | ) { |
| 1215 | const machineId = current.id!; |
| 1216 | const currentVersion = current.instance_id; |
| 1217 | const previous = current; |
| 1218 | const expected = { ...previous, config }; |
| 1219 | yield* leases.mutate(machineId, (lease_nonce) => |
| 1220 | machines |
| 1221 | .updateMachine({ |
| 1222 | app_name: input.appName, |
| 1223 | machine_id: machineId, |
| 1224 | current_version: currentVersion, |
| 1225 | config, |
| 1226 | skip_launch: input.skipLaunch === true ? true : undefined, |
| 1227 | min_secrets_version: input.minSecretsVersion, |
| 1228 | lease_nonce, |
| 1229 | }) |
| 1230 | .pipe(Effect.timeout("30 seconds")), |
| 1231 | ); |
| 1232 | // Rolling opt-out removes generation metadata; observe that transition before startup. |
| 1233 | current = yield* leases.guard( |
| 1234 | Effect.gen(function* () { |
| 1235 | const observed = yield* machines |
| 1236 | .getMachine({ app_name: input.appName, machine_id: machineId }) |
| 1237 | .pipe(Retry.none); |
| 1238 | if (!sameOwnership(observed, expected) && !sameOwnership(observed, previous)) |
| 1239 | return yield* new ReplicaOwnershipChanged({ appName: input.appName, machineId }); |
| 1240 | if ( |
| 1241 | !sameOwnership(observed, expected) || |
| 1242 | input.configDrifted(observed, { mounts, metadata }) || |
| 1243 | !sameChecks(observed.config?.checks, config.checks) || |
| 1244 | !sameStopConfig(observed.config?.stop_config, config.stop_config) |
| 1245 | ) |
| 1246 | return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name }); |
| 1247 | return observed; |
| 1248 | }).pipe( |
| 1249 | Effect.retry({ |
| 1250 | times: 8, |
| 1251 | schedule: waitBackoff, |
| 1252 | while: (error) => |
| 1253 | error._tag === "Fly.ReplicaNotCreated" || |
| 1254 | TRANSIENT_GET_TAGS.some((tag) => tag === error._tag), |
| 1255 | }), |
| 1256 | Effect.timeout("60 seconds"), |
| 1257 | ), |
| 1258 | ); |
| 1259 | } |
| 1260 | |
| 1261 | current = yield* ensureStarted( |
| 1262 | input.appName, |
| 1263 | current, |
| 1264 | input.skipLaunch === true, |
| 1265 | input.policy.healthTimeoutMs, |
| 1266 | config, |
| 1267 | leases, |
| 1268 | ); |
| 1269 | if (current.cordoned === true && !input.skipLaunch) { |
| 1270 | yield* setRouting(input.appName, current.id!, false, leases); |
| 1271 | if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds"); |
| 1272 | current = yield* waitHealthy(input.appName, current, input.policy.healthTimeoutMs, config); |
| 1273 | } |
| 1274 | live.push(current); |
| 1275 | } |
| 1276 | |
| 1277 | const liveIds = new Set(live.map((machine) => machine.id)); |
| 1278 | yield* retireMachines( |
| 1279 | input.appName, |
| 1280 | owned.filter((machine) => !liveIds.has(machine.id)), |
| 1281 | false, |
| 1282 | leases, |
| 1283 | ); |
| 1284 | |
| 1285 | for (const group of groups) { |
| 1286 | for (const extra of [...group.byRegion.values()].flatMap((placed) => placed.extras)) { |
| 1287 | const volumeId = extra.id; |
| 1288 | if (volumeId === undefined || usedVolumeIds.has(volumeId)) continue; |
| 1289 | yield* deleteVolume(input.appName, volumeId); |
| 1290 | } |
| 1291 | } |
| 1292 | |
| 1293 | const volumesById = new Map<string, FlyVolume>(); |
| 1294 | for (const group of groups) { |
| 1295 | for (const volume of [...group.byRegion.values()].flatMap((placed) => placed.volumes)) { |
| 1296 | if (volume.id !== undefined) volumesById.set(volume.id, volume); |
| 1297 | } |
| 1298 | } |
| 1299 | const fresh = yield* Effect.forEach( |
| 1300 | live, |
| 1301 | (machine) => |
| 1302 | machine.id === undefined |
| 1303 | ? Effect.succeed(machine) |
| 1304 | : getMachineById(input.appName, machine.id).pipe(Effect.map((next) => next ?? machine)), |
| 1305 | { concurrency: 4 }, |
| 1306 | ); |
| 1307 | const replicas = fresh.map((machine) => toReplica(machine, volumesById)); |
| 1308 | return toReplicaSet(replicas, input.appName, input.baseName, fresh[0]?.config?.services); |
| 1309 | }); |
| 1310 | |
| 1311 | export const deleteReplicaSet = Effect.fn(function* (input: { |
| 1312 | appName: string; |
| 1313 | id: string; |
| 1314 | type: FlyAlchemyType; |
| 1315 | fqn: string; |
| 1316 | resourceInstanceId: string; |
| 1317 | force?: boolean; |
| 1318 | machineIds: readonly string[]; |
| 1319 | volumeIds: readonly string[]; |
| 1320 | }) { |
| 1321 | return yield* usingMachineLeases(input.appName, undefined, (leases) => |
| 1322 | Effect.gen(function* () { |
| 1323 | const owned = yield* leaseSnapshot( |
| 1324 | input.appName, |
| 1325 | ownedReplicas(yield* listMachinesByApp(input.appName), { |
| 1326 | ...input, |
| 1327 | metadata: yield* createMachineMetadata(input.id, input.type), |
| 1328 | }), |
| 1329 | leases, |
| 1330 | ); |
| 1331 | if (input.force) { |
| 1332 | yield* Effect.forEach( |
| 1333 | owned, |
| 1334 | (machine) => |
| 1335 | Effect.gen(function* () { |
| 1336 | if (!machine.id) return; |
| 1337 | const current = yield* getMachineById(input.appName, machine.id); |
| 1338 | if (!current) return; |
| 1339 | if (!sameOwnership(current, machine)) |
| 1340 | return yield* new ReplicaOwnershipChanged({ |
| 1341 | appName: input.appName, |
| 1342 | machineId: machine.id, |
| 1343 | }); |
| 1344 | yield* deleteMachine(input.appName, machine.id, leases); |
| 1345 | }), |
| 1346 | { concurrency: 4 }, |
| 1347 | ); |
| 1348 | } else yield* retireMachines(input.appName, owned, false, leases); |
| 1349 | yield* Effect.forEach( |
| 1350 | [...new Set(input.volumeIds)].filter((id) => id.length > 0), |
| 1351 | (volumeId) => deleteVolume(input.appName, volumeId), |
| 1352 | { concurrency: 4 }, |
| 1353 | ); |
| 1354 | }), |
| 1355 | ); |
| 1356 | }); |
| 1357 | |
| 1358 | export const volumeIdsOf = (set: { |
| 1359 | mounts?: readonly MountedDisk[]; |
| 1360 | replicas?: readonly Replica[]; |
| 1361 | }): string[] => { |
| 1362 | const ids = new Set<string>(); |
| 1363 | for (const mount of set.mounts ?? []) ids.add(mount.volumeId); |
| 1364 | for (const replica of set.replicas ?? []) { |
| 1365 | for (const mount of replica.mounts) ids.add(mount.volumeId); |
| 1366 | } |
| 1367 | return [...ids]; |
| 1368 | }; |
| 1369 | |
| 1370 | export const groupGenerations = (machines: FlyMachine[]) => { |
| 1371 | const groups = new Map<string | undefined, FlyMachine[]>(); |
| 1372 | for (const machine of machines) { |
| 1373 | const generation = machine.config?.metadata?.[alchemyMetadataKeys.generation]; |
| 1374 | const group = groups.get(generation) ?? []; |
| 1375 | group.push(machine); |
| 1376 | groups.set(generation, group); |
| 1377 | } |
| 1378 | return groups; |
| 1379 | }; |
| 1380 | |
| 1381 | export const observeReplicaSet = Effect.fn(function* (input: { |
| 1382 | appName?: string; |
| 1383 | fqn: string; |
| 1384 | resourceInstanceId: string; |
| 1385 | id: string; |
| 1386 | type: FlyAlchemyType; |
| 1387 | machineIds?: readonly string[]; |
| 1388 | baseName?: string; |
| 1389 | }) { |
| 1390 | if (input.appName === undefined) return undefined; |
| 1391 | const owned = ownedReplicas(yield* listMachinesByApp(input.appName), { |
| 1392 | ...input, |
| 1393 | metadata: yield* createMachineMetadata(input.id, input.type), |
| 1394 | }); |
| 1395 | const groups = groupGenerations(owned); |
| 1396 | const committed = [...groups.entries()] |
| 1397 | .filter(([generation, group]) => { |
| 1398 | if (generation === undefined) return false; |
| 1399 | const count = Number(group[0]?.config?.metadata?.[alchemyMetadataKeys.count]); |
| 1400 | if (group.some((machine) => classifyDeploymentState(machine).protocol === "invalid")) |
| 1401 | return false; |
| 1402 | const protocol2 = group.some( |
| 1403 | (machine) => machine.config?.metadata?.[alchemyMetadataKeys.protocol] === "2", |
| 1404 | ); |
| 1405 | if (protocol2 && !validProtocol2Generation(group)) return false; |
| 1406 | return ( |
| 1407 | Number.isSafeInteger(count) && |
| 1408 | count > 0 && |
| 1409 | group.length === count && |
| 1410 | new Set(group.map(replicaIndexOf)).size === count && |
| 1411 | group.every((machine) => { |
| 1412 | const metadata = machine.config?.metadata; |
| 1413 | const index = replicaIndexOf(machine); |
| 1414 | return ( |
| 1415 | index >= 0 && |
| 1416 | index < count && |
| 1417 | Number(metadata?.[alchemyMetadataKeys.count]) === count && |
| 1418 | metadata?.[alchemyMetadataKeys.phase] === "active" && |
| 1419 | machine.cordoned === false && |
| 1420 | (metadata[alchemyMetadataKeys.protocol] === undefined || |
| 1421 | ((metadata[alchemyMetadataKeys.protocol] === "1" || |
| 1422 | metadata[alchemyMetadataKeys.protocol] === "2") && |
| 1423 | metadata[alchemyMetadataKeys.restored] === "true" && |
| 1424 | (metadata[alchemyMetadataKeys.role] === "idle" || |
| 1425 | (machine.instance_id !== undefined && |
| 1426 | metadata[alchemyMetadataKeys.checkedInstance] === machine.instance_id)))) |
| 1427 | ); |
| 1428 | }) |
| 1429 | ); |
| 1430 | }) |
| 1431 | .sort( |
| 1432 | ([, a], [, b]) => |
| 1433 | Number(b[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0) - |
| 1434 | Number(a[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0), |
| 1435 | ); |
| 1436 | const legacy = (groups.get(undefined) ?? []).filter((machine) => { |
| 1437 | const { protocol } = classifyDeploymentState(machine); |
| 1438 | return protocol === "legacy" || protocol === "1"; |
| 1439 | }); |
| 1440 | const listed = (committed[0]?.[1] ?? legacy).sort( |
| 1441 | (a, b) => replicaIndexOf(a) - replicaIndexOf(b), |
| 1442 | ); |
| 1443 | const rolloutPending = owned.some((machine) => !listed.includes(machine)); |
| 1444 | if (listed.length > 0) { |
| 1445 | const volumesById = new Map<string, FlyVolume>(); |
| 1446 | for (const machine of listed) { |
| 1447 | for (const mount of machine.config?.mounts ?? []) { |
| 1448 | const volumeId = mount.volume; |
| 1449 | if (volumeId === undefined || volumesById.has(volumeId)) continue; |
| 1450 | const volume = yield* getVolumeById(input.appName, volumeId); |
| 1451 | if (volume !== undefined) volumesById.set(volumeId, volume); |
| 1452 | } |
| 1453 | } |
| 1454 | const replicas = listed.map((machine) => toReplica(machine, volumesById)); |
| 1455 | return { |
| 1456 | ...toReplicaSet( |
| 1457 | replicas, |
| 1458 | input.appName, |
| 1459 | listed[0]?.config?.metadata?.[alchemyMetadataKeys.baseName] ?? |
| 1460 | input.baseName ?? |
| 1461 | replicas[0]?.name ?? |
| 1462 | "", |
| 1463 | listed[0]?.config?.services, |
| 1464 | ), |
| 1465 | rolloutPending, |
| 1466 | }; |
| 1467 | } |
| 1468 | if (owned.length) |
| 1469 | return { |
| 1470 | ...toReplicaSet([], input.appName, input.baseName ?? ""), |
| 1471 | region: owned[0]?.region ?? "", |
| 1472 | regions: [ |
| 1473 | ...new Set( |
| 1474 | owned.map((machine) => machine.region ?? "").filter((region) => region.length > 0), |
| 1475 | ), |
| 1476 | ], |
| 1477 | rolloutPending: true, |
| 1478 | }; |
| 1479 | return undefined; |
| 1480 | }); |