The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { expect } from "@alchemy.run/test"; |
| 2 | import type { ScratchStack } from "@alchemy.run/test/Alchemy"; |
| 3 | import { scratchStack } from "@alchemy.run/test/Core"; |
| 4 | import type * as machines from "@distilled.cloud/fly-io/machines"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as HttpClient from "effect/http/HttpClient"; |
| 7 | import * as Layer from "effect/Layer"; |
| 8 | import { throughProxy, transportProxy, type TransportEvent } from "./transport.ts"; |
| 9 | |
| 10 | const metadataKeys = [ |
| 11 | "alchemy.stack", |
| 12 | "alchemy.stage", |
| 13 | "alchemy.id", |
| 14 | "alchemy.type", |
| 15 | "alchemy.replica", |
| 16 | "alchemy.instance", |
| 17 | "alchemy.fqn", |
| 18 | "alchemy.base-name", |
| 19 | "alchemy.generation", |
| 20 | "alchemy.workload", |
| 21 | "alchemy.sequence", |
| 22 | "alchemy.count", |
| 23 | "alchemy.image", |
| 24 | "alchemy.phase", |
| 25 | "alchemy.deployment-protocol", |
| 26 | "alchemy.readiness-roles", |
| 27 | "alchemy.readiness-role", |
| 28 | "alchemy.predecessors", |
| 29 | "alchemy.idle-policy-restored", |
| 30 | "alchemy.checked-instance", |
| 31 | "alchemy.min-secrets-version", |
| 32 | ]; |
| 33 | |
| 34 | const record = (value: unknown): Record<string, unknown> => |
| 35 | typeof value === "object" && value !== null && !Array.isArray(value) |
| 36 | ? (value as Record<string, unknown>) |
| 37 | : {}; |
| 38 | const string = (value: unknown) => (typeof value === "string" ? value : undefined); |
| 39 | const metadataOf = (value: unknown) => { |
| 40 | const metadata = record(value); |
| 41 | return Object.fromEntries( |
| 42 | metadataKeys.flatMap((key) => |
| 43 | typeof metadata[key] === "string" ? [[key, metadata[key] as string]] : [], |
| 44 | ), |
| 45 | ); |
| 46 | }; |
| 47 | const servicesOf = (value: unknown) => |
| 48 | Array.isArray(value) |
| 49 | ? value.map((item) => { |
| 50 | const service = record(item); |
| 51 | return { |
| 52 | protocol: string(service.protocol), |
| 53 | port: typeof service.internal_port === "number" ? service.internal_port : undefined, |
| 54 | autostop: |
| 55 | service.autostop === true |
| 56 | ? "stop" |
| 57 | : service.autostop === "stop" || service.autostop === "suspend" |
| 58 | ? service.autostop |
| 59 | : "off", |
| 60 | autostart: typeof service.autostart === "boolean" ? service.autostart : undefined, |
| 61 | floor: |
| 62 | typeof service.min_machines_running === "number" |
| 63 | ? service.min_machines_running |
| 64 | : undefined, |
| 65 | ...(Array.isArray(service.checks) |
| 66 | ? { |
| 67 | checks: service.checks.map((value) => { |
| 68 | const check = record(value); |
| 69 | return { |
| 70 | type: string(check.type), |
| 71 | port: typeof check.port === "number" ? check.port : undefined, |
| 72 | interval: string(check.interval), |
| 73 | timeout: string(check.timeout), |
| 74 | gracePeriod: string(check.grace_period), |
| 75 | method: string(check.method), |
| 76 | protocol: string(check.protocol), |
| 77 | }; |
| 78 | }), |
| 79 | } |
| 80 | : {}), |
| 81 | }; |
| 82 | }) |
| 83 | : []; |
| 84 | |
| 85 | export interface ReadinessEvent extends TransportEvent { |
| 86 | metadata?: Record<string, string>; |
| 87 | metadataOnly?: boolean; |
| 88 | skipLaunch?: boolean; |
| 89 | services?: ReturnType<typeof servicesOf>; |
| 90 | serviceConfig?: string; |
| 91 | } |
| 92 | |
| 93 | /** Only these two excluded suites install this observer; bodies and headers are never retained. */ |
| 94 | export const readinessProxy = () => |
| 95 | transportProxy().pipe( |
| 96 | Effect.map((proxy) => ({ ...proxy, readiness: [] as ReadinessEvent[] })), |
| 97 | Effect.tap((proxy) => |
| 98 | Effect.addFinalizer(() => |
| 99 | Effect.forEach( |
| 100 | Array.from({ length: Math.ceil(proxy.readiness.length / 50) }, (_, index) => index), |
| 101 | (index) => |
| 102 | Effect.logInfo("Idle/cadence readiness journal", { |
| 103 | offset: index * 50, |
| 104 | events: proxy.readiness.slice(index * 50, (index + 1) * 50), |
| 105 | }), |
| 106 | { discard: true }, |
| 107 | ), |
| 108 | ), |
| 109 | ), |
| 110 | ); |
| 111 | |
| 112 | type ReadinessProxy = Effect.Success<ReturnType<typeof readinessProxy>>; |
| 113 | |
| 114 | export const readinessActor = ( |
| 115 | parent: ScratchStack, |
| 116 | title: string, |
| 117 | file: string, |
| 118 | proxy: ReadinessProxy, |
| 119 | beforeRequest?: (event: ReadinessEvent) => Effect.Effect<void>, |
| 120 | ) => |
| 121 | Effect.sync(() => { |
| 122 | const observer = Layer.effect( |
| 123 | HttpClient.HttpClient, |
| 124 | Effect.gen(function* () { |
| 125 | const client = yield* HttpClient.HttpClient; |
| 126 | return HttpClient.transform(client, (response, request) => |
| 127 | Effect.gen(function* () { |
| 128 | const event = yield* Effect.sync(() => { |
| 129 | const path = new URL(request.url).pathname; |
| 130 | if (!/\/apps\/[^/]+\/machines(?:\/|$)/.test(path)) return undefined; |
| 131 | const event: ReadinessEvent = { |
| 132 | sequence: proxy.readiness.length, |
| 133 | stage: "request", |
| 134 | method: request.method, |
| 135 | path, |
| 136 | machineId: path.match(/\/machines\/([^/]+)/)?.[1], |
| 137 | }; |
| 138 | if (request.body._tag === "Uint8Array") { |
| 139 | try { |
| 140 | const body = record(JSON.parse(new TextDecoder().decode(request.body.body))); |
| 141 | const config = record(body.config); |
| 142 | const metadata = record(body.metadata ?? config.metadata); |
| 143 | event.metadata = metadataOf(metadata); |
| 144 | event.phase = event.metadata["alchemy.phase"]; |
| 145 | event.skipLaunch = |
| 146 | typeof body.skip_launch === "boolean" ? body.skip_launch : undefined; |
| 147 | event.metadataOnly = |
| 148 | Object.keys(body).length === 1 && |
| 149 | "metadata" in body && |
| 150 | Object.keys(metadata).every( |
| 151 | (key) => metadataKeys.includes(key) && typeof metadata[key] === "string", |
| 152 | ); |
| 153 | } catch { |
| 154 | // An unreadable body cannot qualify for the bookkeeping exemption. |
| 155 | } |
| 156 | } |
| 157 | proxy.readiness.push(event); |
| 158 | return event; |
| 159 | }); |
| 160 | if (event && beforeRequest) yield* beforeRequest(event); |
| 161 | const result = yield* response; |
| 162 | if (event) { |
| 163 | const value = |
| 164 | result.status >= 200 && |
| 165 | result.status < 300 && |
| 166 | /\/machines(?:\/[^/]+)?$/.test(event.path) |
| 167 | ? yield* result.json.pipe(Effect.catch(() => Effect.succeed(undefined))) |
| 168 | : undefined; |
| 169 | yield* Effect.sync(() => { |
| 170 | const machine = record(value); |
| 171 | const config = record(machine.config); |
| 172 | const observed: ReadinessEvent = { |
| 173 | ...event, |
| 174 | // Receipt here is after the unmodified proxy forwarded the response. |
| 175 | stage: "forwarded", |
| 176 | status: result.status, |
| 177 | }; |
| 178 | if (typeof machine.id === "string") { |
| 179 | observed.machineId = machine.id; |
| 180 | observed.instanceId = string(machine.instance_id); |
| 181 | observed.state = string(machine.state); |
| 182 | observed.cordoned = |
| 183 | typeof machine.cordoned === "boolean" ? machine.cordoned : undefined; |
| 184 | observed.digest = string(record(machine.image_ref).digest); |
| 185 | observed.metadata = metadataOf(config.metadata); |
| 186 | observed.phase = observed.metadata["alchemy.phase"]; |
| 187 | observed.services = servicesOf(config.services); |
| 188 | observed.serviceConfig = JSON.stringify(observed.services); |
| 189 | observed.checks = Array.isArray(machine.checks) |
| 190 | ? machine.checks.map((item) => { |
| 191 | const check = record(item); |
| 192 | return { |
| 193 | name: string(check.name), |
| 194 | status: string(check.status), |
| 195 | }; |
| 196 | }) |
| 197 | : []; |
| 198 | } |
| 199 | proxy.readiness.push(observed); |
| 200 | }); |
| 201 | } |
| 202 | return result; |
| 203 | }), |
| 204 | ); |
| 205 | }), |
| 206 | ).pipe(Layer.provideMerge(throughProxy(() => proxy.url))); |
| 207 | const actor = scratchStack({ providers: observer, stage: parent.stage }, title, file); |
| 208 | expect(actor.name).toBe(parent.name); |
| 209 | expect(actor.stage).toBe(parent.stage); |
| 210 | expect(actor.state).not.toBe(parent.state); |
| 211 | return actor; |
| 212 | }); |
| 213 | |
| 214 | export const retires = (event: TransportEvent, ids: readonly string[]) => |
| 215 | event.stage === "request" && |
| 216 | ids.includes(event.machineId!) && |
| 217 | (event.path.endsWith("/cordon") || |
| 218 | event.path.endsWith("/stop") || |
| 219 | event.path.endsWith("/suspend") || |
| 220 | (event.path.endsWith("/metadata") && event.phase === "retiring") || |
| 221 | (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path))); |
| 222 | |
| 223 | const identity = (metadata: Record<string, string> | undefined) => |
| 224 | Object.fromEntries( |
| 225 | Object.entries(metadata ?? {}).filter( |
| 226 | ([key]) => key !== "alchemy.phase" && key !== "alchemy.checked-instance", |
| 227 | ), |
| 228 | ); |
| 229 | |
| 230 | export const readinessChecksPassing = ( |
| 231 | checks: ReadinessEvent["checks"], |
| 232 | expected: readonly string[], |
| 233 | ) => { |
| 234 | if (!checks || expected.length === 0 || new Set(expected).size !== expected.length) return false; |
| 235 | const allowed = new Set([ |
| 236 | ...expected, |
| 237 | ...expected |
| 238 | .filter((name) => name.startsWith("servicecheck-")) |
| 239 | .map((name) => `bg_deployments_compat-${name.slice("servicecheck-".length)}`), |
| 240 | ]); |
| 241 | const reported = new Set(checks.map((check) => check.name)); |
| 242 | return ( |
| 243 | reported.size === checks.length && |
| 244 | expected.every((name) => reported.has(name)) && |
| 245 | checks.every( |
| 246 | (check) => check.name !== undefined && allowed.has(check.name) && check.status === "passing", |
| 247 | ) |
| 248 | ); |
| 249 | }; |
| 250 | |
| 251 | const sameMetadata = (left: ReadinessEvent, right: ReadinessEvent) => |
| 252 | left.method === "PATCH" && |
| 253 | right.method === "PATCH" && |
| 254 | left.path.endsWith("/metadata") && |
| 255 | left.path === right.path && |
| 256 | left.metadataOnly === true && |
| 257 | right.metadataOnly === true && |
| 258 | left.metadata !== undefined && |
| 259 | right.metadata !== undefined && |
| 260 | JSON.stringify(Object.entries(left.metadata).sort()) === |
| 261 | JSON.stringify(Object.entries(right.metadata).sort()); |
| 262 | |
| 263 | /** Fresh GET receipt proves observation order, not the remote check sample's timestamp. */ |
| 264 | export const assertReadinessCommit = ( |
| 265 | events: readonly ReadinessEvent[], |
| 266 | priorIds: readonly string[], |
| 267 | candidates: machines.Machine[], |
| 268 | runningSlots: readonly number[], |
| 269 | checkNames: readonly string[], |
| 270 | allowIdle: boolean, |
| 271 | ) => { |
| 272 | const retirement = |
| 273 | priorIds.length > 0 ? events.findIndex((event) => retires(event, priorIds)) : events.length; |
| 274 | expect(retirement).toBeGreaterThan(0); |
| 275 | const ordered = [...candidates].sort( |
| 276 | (a, b) => |
| 277 | Number(a.config?.metadata?.["alchemy.replica"]) - |
| 278 | Number(b.config?.metadata?.["alchemy.replica"]), |
| 279 | ); |
| 280 | const ids = ordered.map((machine) => machine.id!); |
| 281 | const creates = events |
| 282 | .filter( |
| 283 | (event) => |
| 284 | event.stage === "forwarded" && |
| 285 | event.method === "POST" && |
| 286 | event.path.endsWith("/machines") && |
| 287 | ids.includes(event.machineId!), |
| 288 | ) |
| 289 | .map((event) => event.sequence); |
| 290 | const mutations = events.filter( |
| 291 | (event) => |
| 292 | event.stage === "request" && |
| 293 | event.method !== "GET" && |
| 294 | !event.path.endsWith("/lease") && |
| 295 | (ids.includes(event.machineId!) || creates.includes(event.sequence)), |
| 296 | ); |
| 297 | // Phase stamps go through Fly's `PATCH /machines/{id}/metadata` (formerly PUT). |
| 298 | const isCommit = (event: ReadinessEvent) => |
| 299 | event.method === "PATCH" && |
| 300 | event.path.endsWith("/metadata") && |
| 301 | event.phase === "active" && |
| 302 | event.metadataOnly === true; |
| 303 | const isValidating = (event: ReadinessEvent) => |
| 304 | event.method === "PATCH" && |
| 305 | event.path.endsWith("/metadata") && |
| 306 | event.phase === "validating" && |
| 307 | event.metadataOnly === true; |
| 308 | const successful = (event: ReadinessEvent) => event.status! >= 200 && event.status! < 300; |
| 309 | const operations: { |
| 310 | request: ReadinessEvent; |
| 311 | machineId: string; |
| 312 | completed: number; |
| 313 | }[] = []; |
| 314 | for (const request of mutations) { |
| 315 | const receipts = events.filter( |
| 316 | (event) => |
| 317 | event.sequence === request.sequence && |
| 318 | event.stage === "forwarded" && |
| 319 | event.method === request.method && |
| 320 | event.path === request.path, |
| 321 | ); |
| 322 | expect(receipts).toHaveLength(1); |
| 323 | const receipt = receipts[0]!; |
| 324 | const completed = events.indexOf(receipt); |
| 325 | const requested = events.indexOf(request); |
| 326 | expect(completed).toBeGreaterThan(requested); |
| 327 | expect(completed).toBeLessThan(retirement); |
| 328 | const machineId = request.machineId ?? receipt.machineId!; |
| 329 | if (request.path.endsWith("/start") && receipt.status === 412) { |
| 330 | const candidate = ordered.find((machine) => machine.id === machineId)!; |
| 331 | const commit = events.findIndex((event) => isCommit(event) && event.machineId === machineId); |
| 332 | expect( |
| 333 | events.some( |
| 334 | (event, index) => |
| 335 | index > completed && |
| 336 | index < commit && |
| 337 | event.stage === "forwarded" && |
| 338 | event.method === "GET" && |
| 339 | successful(event) && |
| 340 | event.machineId === machineId && |
| 341 | event.state === "started" && |
| 342 | event.instanceId === candidate.instance_id, |
| 343 | ), |
| 344 | ).toBe(true); |
| 345 | continue; |
| 346 | } |
| 347 | const previous = operations.findLast((operation) => operation.machineId === machineId); |
| 348 | // Only settled, retryable metadata failures may join an equivalent retry. |
| 349 | if ( |
| 350 | previous && |
| 351 | !successful(events[previous.completed]!) && |
| 352 | sameMetadata(previous.request, request) |
| 353 | ) { |
| 354 | expect([429, 500, 502, 503, 504]).toContain(events[previous.completed]!.status); |
| 355 | expect(previous.completed).toBeLessThan(requested); |
| 356 | previous.completed = completed; |
| 357 | } else { |
| 358 | operations.push({ request, machineId, completed }); |
| 359 | } |
| 360 | } |
| 361 | for (const operation of operations) { |
| 362 | expect(successful(events[operation.completed]!)).toBe(true); |
| 363 | } |
| 364 | const commits = operations.filter((operation) => isCommit(operation.request)); |
| 365 | expect(commits).toHaveLength(ordered.length); |
| 366 | expect(commits.map((operation) => operation.machineId).sort()).toEqual([...ids].sort()); |
| 367 | expect(commits.at(-1)?.machineId).toBe(ordered[0]!.id); |
| 368 | const firstCommit = events.indexOf(commits[0]!.request); |
| 369 | expect(firstCommit).toBeGreaterThan(0); |
| 370 | const invalidating = operations.filter((operation) => !isCommit(operation.request)); |
| 371 | expect(invalidating.length).toBeGreaterThan(0); |
| 372 | const reset = Math.max(...invalidating.map((operation) => operation.completed)); |
| 373 | expect(reset).toBeLessThan(firstCommit); |
| 374 | const zeroCommit = events.indexOf(commits.at(-1)!.request); |
| 375 | for (const operation of commits.slice(0, -1)) { |
| 376 | expect(operation.completed).toBeLessThan(zeroCommit); |
| 377 | } |
| 378 | |
| 379 | for (const [slot, machine] of ordered.entries()) { |
| 380 | const id = machine.id!; |
| 381 | const run = runningSlots.includes(slot); |
| 382 | const commit = commits.find((operation) => operation.machineId === id)!.request; |
| 383 | const finalMetadata = metadataOf(machine.config?.metadata); |
| 384 | const finalServices = servicesOf(machine.config?.services); |
| 385 | if (allowIdle) { |
| 386 | expect(finalServices.length).toBeGreaterThan(0); |
| 387 | expect(finalServices.every((service) => service.autostop !== "off")).toBe(true); |
| 388 | } |
| 389 | expect(finalMetadata["alchemy.replica"]).toBe(String(slot)); |
| 390 | expect(finalMetadata["alchemy.readiness-role"]).toBe(run ? "run" : "idle"); |
| 391 | expect(finalMetadata["alchemy.readiness-roles"]).toBe( |
| 392 | ordered.map((_, index) => (runningSlots.includes(index) ? "run" : "idle")).join(","), |
| 393 | ); |
| 394 | expect(finalMetadata["alchemy.phase"]).toBe("active"); |
| 395 | expect(finalMetadata["alchemy.idle-policy-restored"]).toBe("true"); |
| 396 | expect(commit.metadata).toEqual(finalMetadata); |
| 397 | expect(machine.cordoned).toBe(false); |
| 398 | for (const key of [ |
| 399 | "alchemy.stack", |
| 400 | "alchemy.stage", |
| 401 | "alchemy.id", |
| 402 | "alchemy.type", |
| 403 | "alchemy.instance", |
| 404 | "alchemy.fqn", |
| 405 | "alchemy.generation", |
| 406 | "alchemy.workload", |
| 407 | "alchemy.image", |
| 408 | ]) { |
| 409 | expect(finalMetadata[key]).toBeDefined(); |
| 410 | } |
| 411 | const observations = events.flatMap((event, index) => { |
| 412 | if ( |
| 413 | index >= firstCommit || |
| 414 | event.stage !== "forwarded" || |
| 415 | event.method !== "GET" || |
| 416 | !event.path.endsWith(`/machines/${id}`) || |
| 417 | event.machineId !== id || |
| 418 | event.status !== 200 |
| 419 | ) |
| 420 | return []; |
| 421 | const requested = events.findIndex( |
| 422 | (request) => request.stage === "request" && request.sequence === event.sequence, |
| 423 | ); |
| 424 | return requested >= 0 && requested < index ? [{ event, index, requested }] : []; |
| 425 | }); |
| 426 | const pending = observations.findLast(({ requested }) => requested > reset); |
| 427 | expect(pending).toBeDefined(); |
| 428 | if (!pending) continue; |
| 429 | expect(pending.event.phase).toBe("validating"); |
| 430 | expect(pending.event.metadata?.["alchemy.idle-policy-restored"]).toBe("true"); |
| 431 | expect(pending.event.instanceId).toBe(machine.instance_id); |
| 432 | let proof = pending; |
| 433 | if (run && pending.event.state !== "started") { |
| 434 | expect(allowIdle).toBe(true); |
| 435 | expect(["stopped", "suspended"]).toContain(pending.event.state); |
| 436 | const targetReset = Math.max( |
| 437 | -1, |
| 438 | ...invalidating |
| 439 | .filter((operation) => operation.machineId === id && !isValidating(operation.request)) |
| 440 | .map((operation) => operation.completed), |
| 441 | ); |
| 442 | const restored = observations.findLast( |
| 443 | ({ event, requested, index }) => |
| 444 | requested > targetReset && |
| 445 | index < pending.index && |
| 446 | ["promoting", "validating"].includes(event.phase ?? "") && |
| 447 | event.metadata?.["alchemy.idle-policy-restored"] === "true" && |
| 448 | event.instanceId === pending.event.instanceId && |
| 449 | event.state === "started" && |
| 450 | readinessChecksPassing(event.checks, checkNames), |
| 451 | ); |
| 452 | expect(restored).toBeDefined(); |
| 453 | if (!restored) continue; |
| 454 | proof = restored; |
| 455 | } else if (run) { |
| 456 | expect(readinessChecksPassing(proof.event.checks, checkNames)).toBe(true); |
| 457 | } else { |
| 458 | expect(["created", "stopped", "suspended"]).toContain(pending.event.state); |
| 459 | } |
| 460 | expect(proof.event.instanceId).toBeDefined(); |
| 461 | expect(proof.event.digest).toBe(machine.image_ref?.digest); |
| 462 | expect(proof.event.services).toEqual(finalServices); |
| 463 | expect(proof.event.cordoned).toBe(false); |
| 464 | expect(identity(proof.event.metadata)).toEqual(identity(finalMetadata)); |
| 465 | expect(commit.metadata).toEqual({ |
| 466 | ...pending.event.metadata, |
| 467 | "alchemy.phase": "active", |
| 468 | ...(run ? { "alchemy.checked-instance": proof.event.instanceId } : {}), |
| 469 | }); |
| 470 | if (run) { |
| 471 | expect(commit.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId); |
| 472 | expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain( |
| 473 | machine.state, |
| 474 | ); |
| 475 | } else { |
| 476 | expect(["created", "stopped", "suspended"]).toContain(machine.state); |
| 477 | for (const created of events.filter( |
| 478 | (event) => |
| 479 | event.stage === "forwarded" && |
| 480 | event.method === "POST" && |
| 481 | event.path.endsWith("/machines") && |
| 482 | event.machineId === id, |
| 483 | )) { |
| 484 | expect(created.skipLaunch).toBe(true); |
| 485 | } |
| 486 | expect( |
| 487 | events.some( |
| 488 | (event) => |
| 489 | event.machineId === id && |
| 490 | ((event.stage === "request" && event.path.endsWith("/start")) || |
| 491 | (event.stage === "forwarded" && event.state === "started")), |
| 492 | ), |
| 493 | ).toBe(false); |
| 494 | } |
| 495 | // Idle proof survives only the unchanged pending stamp and final commit. |
| 496 | for (const operation of invalidating.filter( |
| 497 | (operation) => operation.machineId === id && operation.completed > proof.index, |
| 498 | )) { |
| 499 | expect(isValidating(operation.request)).toBe(true); |
| 500 | expect(events.indexOf(operation.request)).toBeGreaterThan(proof.index); |
| 501 | expect(operation.request.metadata).toEqual({ |
| 502 | ...proof.event.metadata, |
| 503 | "alchemy.phase": "validating", |
| 504 | }); |
| 505 | } |
| 506 | for (const later of events |
| 507 | .slice(proof.index + 1) |
| 508 | .filter( |
| 509 | (event) => |
| 510 | event.stage === "forwarded" && |
| 511 | event.machineId === id && |
| 512 | event.status! >= 200 && |
| 513 | event.status! < 300 && |
| 514 | event.path.endsWith(`/machines/${id}`), |
| 515 | )) { |
| 516 | expect(later.instanceId).toBe(proof.event.instanceId); |
| 517 | expect(later.digest).toBe(proof.event.digest); |
| 518 | expect(later.services).toEqual(finalServices); |
| 519 | expect(identity(later.metadata)).toEqual(identity(finalMetadata)); |
| 520 | expect(later.metadata?.["alchemy.idle-policy-restored"]).toBe("true"); |
| 521 | expect(["promoting", "validating", "active"]).toContain(later.phase); |
| 522 | expect(later.cordoned).toBe(false); |
| 523 | if (run) { |
| 524 | expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain( |
| 525 | later.state, |
| 526 | ); |
| 527 | if (later.state === "started" && events.indexOf(later) < firstCommit) { |
| 528 | expect(readinessChecksPassing(later.checks, checkNames)).toBe(true); |
| 529 | } |
| 530 | if (later.phase === "active") { |
| 531 | expect(later.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId); |
| 532 | } |
| 533 | } |
| 534 | } |
| 535 | } |
| 536 | }; |