@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 2,545 lines 96.7 KB
Raw
1 import * as Cause from "effect/Cause";
2 import type { ConfigError } from "effect/Config";
3 import * as Data from "effect/Data";
4 import * as Deferred from "effect/Deferred";
5 import * as Effect from "effect/Effect";
6 import * as Formatter from "effect/Formatter";
7 import * as Layer from "effect/Layer";
8 import * as Option from "effect/Option";
9 import type { PlatformError } from "effect/PlatformError";
10 import * as Predicate from "effect/Predicate";
11 import type { Simplify } from "effect/Types";
12 import type { ActionLike } from "./Action.ts";
13 import { makeResolveContext } from "./ActionRuntimeContext.ts";
14 import {
15 OwnedBySomeoneElse,
16 stripUnowned,
17 Unowned,
18 Unresolved,
19 UnverifiableReplacement,
20 } from "./AdoptPolicy.ts";
21 import { AlchemyContext } from "./AlchemyContext.ts";
22 import {
23 Artifacts,
24 ArtifactStore,
25 createArtifactStore,
26 ensureArtifactStore,
27 makeScopedArtifacts,
28 } from "./Artifacts.ts";
29 import type { AuthError, NeedsReauth } from "./Auth/AuthProvider.ts";
30 import {
31 type CredentialsRequired,
32 failCredentialsRequired,
33 demandPlanCredentials,
34 } from "./Auth/Demand.ts";
35 import { havePropsChanged, isResolved, stripEffects, stripUnresolved } from "./Diff.ts";
36 import { captureEmit, type ApplyEvent, type ApplyStatus } from "./Events.ts";
37 import type { Input } from "./Input.ts";
38 import { generateInstanceId } from "./InstanceId.ts";
39 import * as Output from "./Output.ts";
40 import { type ActionApply, type Apply, type Delete, type Plan } from "./Plan.ts";
41 import {
42 type DeleteInProgress,
43 type ProviderService,
44 findProviderByType,
45 missingProviderError,
46 tryFindProviderByType,
47 } from "./Provider.ts";
48 import { stampedMode, type ProviderMode } from "./ProviderMode.ts";
49 import { resourceSession } from "./Report.ts";
50 import type { ResourceBinding } from "./Resource.ts";
51 import { ResourceContext } from "./ResourceContext.ts";
52 import { RuntimeContext } from "./RuntimeContext.ts";
53 import { Stack } from "./Stack.ts";
54 import { Stage } from "./Stage.ts";
55 import {
56 type ActionState,
57 type CreatedResourceState,
58 type CreatingResourceState,
59 type DeletingResourceState,
60 type PersistedState,
61 type RanActionState,
62 type ReplacedResourceState,
63 type ReplacementOldResourceState,
64 type ReplacementResourceState,
65 type ReplacingResourceState,
66 type ResourceState,
67 type RunningActionState,
68 type UpdatedResourceState,
69 type UpdatingReourceState,
70 State,
71 StateStoreError,
72 } from "./State/index.ts";
73 import { type ResourceOp, recordResourceOp } from "./Telemetry/Metrics.ts";
74 import { LogAnnotation } from "./Util/LogAnnotation.ts";
75 import { hashInput } from "./Util/sha256.ts";
76
77 export type AppliedPlan<P extends Plan> = {
78 [id in keyof P["resources"]]: P["resources"][id] extends Delete | undefined
79 ? never
80 : Simplify<P["resources"][id]["resource"]["attr"]>;
81 };
82
83 /**
84 * Where the engine reports apply progress. Every event goes out as a log
85 * record; renderers observe them with `onEvent`.
86 */
87 type Reporter = (event: ApplyEvent) => Effect.Effect<void>;
88
89 interface ResourceTracker {
90 output: any;
91 props: any;
92 bindings: ResourceBinding[];
93 instanceId: string;
94 }
95
96 /**
97 * Record a resource's result in `tracker` and its output in `outputs`, the
98 * `fqn -> output` record `Output.evaluate` reads. Keeping `outputs` up to date
99 * here means callers never rebuild it from `tracker`, which made apply
100 * quadratic in resource count (#2400).
101 */
102 const track = (
103 tracker: Record<string, ResourceTracker>,
104 outputs: Record<string, any>,
105 fqn: string,
106 entry: ResourceTracker,
107 ) => {
108 tracker[fqn] = entry;
109 outputs[fqn] = entry.output;
110 };
111
112 /**
113 * Whether an attr-less checkpoint lost props to `stripUnresolved`. Its stored
114 * props no longer identify the physical resource it was creating (e.g. a
115 * stripped namespace makes a name match a different resource), so recovery
116 * must not look the resource up from them.
117 *
118 * Effect-valued props don't count: `Output.evaluate` passes them through
119 * untouched, so even fully resolved props carry them (a Function's runtime
120 * wiring). They never name a physical resource, and treating them as holes
121 * would mark every create of such a resource and orphan it on interruption.
122 */
123 const propsIncomplete = (value: {
124 attr?: unknown;
125 props?: unknown;
126 propsIncomplete?: true;
127 }): true | undefined =>
128 value.attr === undefined && (value.propsIncomplete || !isResolved(stripEffects(value.props)))
129 ? true
130 : undefined;
131
132 const provideLifecycleScope =
133 (resource: ResourceContext["Service"]) =>
134 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
135 Effect.serviceOption(ArtifactStore).pipe(
136 Effect.map(Option.getOrElse(createArtifactStore)),
137 Effect.flatMap((store) =>
138 effect.pipe(
139 failCredentialsRequired(resource.fqn),
140 Effect.provide([
141 Layer.succeed(Artifacts, makeScopedArtifacts(store, resource.fqn)),
142 Layer.succeed(ResourceContext, resource),
143 ]),
144 ),
145 ),
146 );
147
148 /**
149 * Instruments a single provider lifecycle call with an OTel span
150 * (`provider.<op>`), the resource counter / duration histogram, and the
151 * scoped artifacts/resource context normally supplied by
152 * {@link provideLifecycleScope}.
153 *
154 * This is the only call site through which provider lifecycle methods
155 * are dispatched, so wrapping it here gives us a fully-instrumented
156 * toolchain without touching any individual provider implementation.
157 */
158 const instrumentLifecycle =
159 (op: ResourceOp, fqn: string, resourceType: string, logicalId: string, instanceId: string) =>
160 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
161 effect.pipe(
162 provideLifecycleScope({ logicalId, fqn, instanceId, type: resourceType }),
163 // Every log a provider writes during the op carries its resource.
164 Effect.annotateLogs(LogAnnotation.fqn, fqn),
165 recordResourceOp(resourceType, op),
166 Effect.withSpan(`provider.${op}`, {
167 attributes: {
168 "alchemy.resource.fqn": fqn,
169 "alchemy.resource.type": resourceType,
170 "alchemy.resource.logical_id": logicalId,
171 "alchemy.resource.instance_id": instanceId,
172 "alchemy.resource.op": op,
173 },
174 }),
175 );
176
177 type AdmissionNode = {
178 provider: Pick<ProviderService<any>, "observe" | "read">;
179 props: any;
180 deferredAdoption?: { adopt: boolean };
181 resource: { Type: string };
182 };
183
184 /**
185 * A dependency cycle precreates on raw props, before `read` can check
186 * ownership. Ask the provider's `observe` what exists at the identity
187 * precreate would claim, and refuse a foreign resource unless adoption is
188 * allowed. A provider without `observe` precreates `unchecked`; one whose
189 * identity has not resolved yet reports it `unresolved`, so the engine must
190 * check the resolved identity before reconcile claims it.
191 */
192 const observeBeforePrecreate = Effect.fn(function* (
193 node: AdmissionNode,
194 news: any,
195 fqn: string,
196 logicalId: string,
197 instanceId: string,
198 previous?: any,
199 ) {
200 if (node.provider.observe === undefined || node.deferredAdoption === undefined) {
201 return "unchecked" as const;
202 }
203 const observed = yield* node.provider
204 .observe({ id: logicalId, fqn, instanceId, news, previous })
205 .pipe(instrumentLifecycle("observe", fqn, node.resource.Type, logicalId, instanceId));
206 if (observed === Unresolved) return "unresolved" as const;
207 if (observed !== undefined && Unowned.is(observed) && !node.deferredAdoption.adopt) {
208 return yield* new OwnedBySomeoneElse({
209 message:
210 `Cannot pre-create resource '${fqn}' (${node.resource.Type}): ` +
211 "it exists in the cloud but is not owned by this " +
212 "stack/stage/logical-id. Re-run with `--adopt` (or " +
213 "wrap the effect in `adopt(true)`) to take it over.",
214 resourceType: node.resource.Type,
215 logicalId,
216 });
217 }
218 return "admitted" as const;
219 });
220
221 /** A provider's `observe` reported `Unresolved` for fully resolved props. */
222 class UnresolvedObservation extends Data.TaggedError("UnresolvedObservation")<{
223 message: string;
224 }> {}
225
226 /**
227 * Reports what exists at the resolved identity reconcile will claim: `read`
228 * when the provider implements it, `observe` otherwise. A replacement passes
229 * its `previous` generation's attributes, so the provider can recognize it.
230 */
231 const observeResolved = Effect.fn(function* (
232 node: AdmissionNode,
233 news: Record<string, any>,
234 fqn: string,
235 logicalId: string,
236 instanceId: string,
237 previous?: any,
238 ) {
239 if (node.provider.read) {
240 return yield* node.provider
241 .read({ id: logicalId, fqn, instanceId, olds: news, output: undefined, previous })
242 .pipe(instrumentLifecycle("read", fqn, node.resource.Type, logicalId, instanceId));
243 }
244 const observed = yield* node.provider.observe!({
245 id: logicalId,
246 fqn,
247 instanceId,
248 news,
249 previous,
250 }).pipe(instrumentLifecycle("observe", fqn, node.resource.Type, logicalId, instanceId));
251 if (observed === Unresolved) {
252 return yield* Effect.die(
253 new UnresolvedObservation({
254 message: `The provider for '${fqn}' (${node.resource.Type}) reported its identity as Unresolved, but its props are resolved.`,
255 }),
256 );
257 }
258 return observed;
259 });
260
261 export const apply = <P extends Plan>(
262 plan: P,
263 ): Effect.Effect<
264 Input.Resolve<P["output"]>,
265 | Output.InvalidReferenceError
266 | Output.MissingSourceError
267 | StateStoreError
268 | DestroyError
269 | CredentialsRequired
270 | AuthError
271 | NeedsReauth
272 | PlatformError
273 | ConfigError,
274 State | Stack | Stage
275 > =>
276 Effect.gen(function* () {
277 // Credential-free dev: a dev-mode plan that needs the real cloud
278 // (`Alchemy.remote()` rows, remote-proxied bindings, deletions of rows
279 // stamped `providerMode: "live"`) demands cloud credentials exactly
280 // once, up front, BEFORE any lifecycle operation runs — a fully-local
281 // dev plan demands nothing. Non-dev runs never enter the seam: live
282 // providers keep the pre-existing lazy credential flow. Wired here (not
283 // in Deploy/Destroy) because `apply` is the single choke point every
284 // path shares — CLI deploy/destroy, `Test.make` deploys, and
285 // `test.provider` scratch stacks. See `Auth/Demand.ts`.
286 const alchemy = yield* Effect.serviceOption(AlchemyContext);
287 if (Option.isSome(alchemy) && alchemy.value.dev) {
288 yield* demandPlanCredentials(plan);
289 }
290
291 const emitEvent = yield* captureEmit;
292 // The last status each resource announced, so the post-convergence
293 // flush below re-emits only what is still unannounced (cycle members).
294 const announced = new Map<string, ApplyStatus>();
295 const reportEvent: Reporter = (event) => {
296 if (event.type === "apply.resource.status") announced.set(event.fqn, event.status);
297 return emitEvent(event);
298 };
299 // No Effect.scoped around apply: local dev providers start long-running
300 // processes in the AMBIENT scope during reconcile, which a scope here
301 // would kill when apply returns.
302 const state = yield* yield* State;
303 const stack = yield* Stack;
304 const stage = yield* Stage;
305 const stackName = stack.name;
306
307 const tracker: Record<string, ResourceTracker> = {};
308 const outputs: Record<string, any> = {};
309 const terminalStatuses = new Map<
310 string,
311 {
312 fqn: string;
313 id: string;
314 type: string;
315 status: Extract<ApplyStatus, "created" | "updated" | "adopted" | "ran" | "skipped">;
316 providerMode?: ProviderMode;
317 }
318 >();
319
320 // ── FQN migrations (renamedFrom) ──
321 // Persist renames before any lifecycle operation runs: a node whose row
322 // was found under a former FQN carries that row pre-remapped in
323 // `node.state` (see Plan's rename resolution). Commit it at the current
324 // FQN FIRST, then drop the former row — in that order, so an
325 // interruption leaves rows at both FQNs with the same instanceId, which
326 // the next plan recognizes as an in-flight migration (and never as an
327 // orphan to delete).
328 //
329 // In a same-deploy shift (A→B while B→C), C's former FQN `B` is
330 // simultaneously B's migration TARGET. C must NOT delete it: B's own
331 // `state.set` supersedes the stale copy, and the migrations run
332 // concurrently — the delete could land after B's write and destroy the
333 // freshly migrated row.
334 const migrationTargets = new Set(
335 Object.values(plan.resources)
336 .filter((node) => node.renamedFrom?.length && node.state !== undefined)
337 .map((node) => node.resource.FQN),
338 );
339 yield* Effect.forEach(
340 Object.values(plan.resources),
341 (node) => {
342 const { renamedFrom, state: row } = node;
343 return renamedFrom === undefined || renamedFrom.length === 0 || row === undefined
344 ? Effect.void
345 : Effect.gen(function* () {
346 yield* state.set({ stack: stackName, stage, fqn: node.resource.FQN, value: row });
347 yield* Effect.forEach(
348 renamedFrom.filter((formerFqn) => !migrationTargets.has(formerFqn)),
349 (formerFqn) => state.delete({ stack: stackName, stage, fqn: formerFqn }),
350 { concurrency: "unbounded" },
351 );
352 });
353 },
354 { concurrency: "unbounded" },
355 );
356
357 yield* executePlan(
358 plan,
359 tracker,
360 outputs,
361 terminalStatuses,
362 reportEvent,
363 state,
364 stackName,
365 stage,
366 );
367
368 // TODO(sam): support roll back to previous state if errors occur during expansion
369 // -> RISK: some UPDATEs may not be reversible (i.e. trigger replacements)
370 // TODO(sam): should pivot be done separately? E.g shift traffic?
371
372 yield* collectGarbage(plan, reportEvent);
373
374 yield* converge(plan, tracker, outputs, terminalStatuses, reportEvent, state, stackName, stage);
375
376 // Cycle members hold their terminal status until `converge` settles
377 // it; everything else announced it as it finished.
378 yield* Effect.forEach(
379 Array.from(terminalStatuses.values()).filter(
380 ({ fqn, status }) => announced.get(fqn) !== status,
381 ),
382 ({ fqn, id, type, status, providerMode }) =>
383 reportEvent({
384 type: "apply.resource.status",
385 fqn,
386 id,
387 resourceType: type,
388 status,
389 providerMode,
390 }),
391 { concurrency: "unbounded" },
392 );
393
394 if (plan.destroy) {
395 // The destroy converged: every resource row was deleted above. Drop
396 // the rest of the stage's persisted state — notably the stack output
397 // record written by the last deploy — so `getOutput` returns
398 // undefined and `listStages` no longer reports the stage.
399 // https://github.com/alchemy-run/alchemy/issues/961
400 yield* state.deleteStack({ stack: stackName, stage });
401 // Invariant: a successful destroy leaves the stage EMPTY. If rows
402 // survive, this destroy session could not actually see (or delete)
403 // the stack's state — e.g. its plan listed an empty store while
404 // committed rows existed — and reporting success here would silently
405 // leak every cloud resource those rows track. Fail loudly instead so
406 // the leak surfaces in the run that caused it.
407 const remaining = yield* state.list({ stack: stackName, stage });
408 if (remaining.length > 0) {
409 return yield* new StateStoreError({
410 message:
411 `destroy of ${stackName}/${stage} reported success but ${remaining.length} ` +
412 `state row(s) remain (${remaining.join(", ")}) — the destroy session could ` +
413 `not see the stack's persisted state, so its cloud resources were NOT deleted`,
414 });
415 }
416 return undefined;
417 }
418
419 if (plan.selectedFqns !== undefined || !plan.output) {
420 return undefined;
421 }
422
423 const resolved = yield* Output.evaluate(plan.output, outputs);
424
425 // Persist the stack's evaluated outputs so cross-stack references
426 // (`yield* OtherStack` / `OtherStack.stage.<name>` / `Output.stackRef`)
427 // can read them back out of the state store.
428 yield* state.setOutput({ stack: stackName, stage, value: resolved });
429
430 return resolved;
431 }).pipe(
432 ensureArtifactStore,
433 Effect.withSpan("apply", {
434 attributes: {
435 "alchemy.resources.count": Object.keys(plan.resources).length,
436 "alchemy.deletions.count": Object.keys(plan.deletions).length,
437 },
438 }),
439 );
440
441 // ── Phase 1: concurrent initial execution ──────────────────────────────────
442 //
443 // Each resource gets a Deferred<void> that signals "I have some output
444 // available in `tracker`." Resources with `precreate` signal early so that
445 // downstream resources can resolve stable identifiers without deadlocking.
446 // The actual output lives in the mutable `tracker` map, not in the Deferred.
447
448 const executePlan = Effect.fn(function* (
449 plan: Plan,
450 tracker: Record<string, ResourceTracker>,
451 outputs: Record<string, any>,
452 terminalStatuses: Map<
453 string,
454 {
455 fqn: string;
456 id: string;
457 type: string;
458 status: Extract<ApplyStatus, "created" | "updated" | "adopted" | "ran" | "skipped">;
459 providerMode?: ProviderMode;
460 }
461 >,
462 reportEvent: Reporter,
463 state: {
464 set: <V extends PersistedState>(req: {
465 stack: string;
466 stage: string;
467 fqn: string;
468 value: V;
469 }) => Effect.Effect<V, StateStoreError, never>;
470 },
471 stackName: string,
472 stage: string,
473 ) {
474 // Resources and tasks share the same FQN namespace and DAG, so the
475 // scheduler tracks them together. Each entry gets a single Deferred that
476 // signals "my output is available in `tracker`."
477 const allNodes: Record<string, Apply | ActionApply> = { ...plan.resources, ...plan.actions };
478
479 const ready = Object.fromEntries(
480 yield* Effect.forEach(Object.keys(allNodes), (fqn) =>
481 Effect.map(Deferred.make<void>(), (d) => [fqn, d] as const),
482 ),
483 ) as Record<string, Deferred.Deferred<void>>;
484
485 // `readyStable` fires only when a node has reached its TERMINAL output —
486 // resources after `reconcile`, tasks after the body completes. Resource
487 // precreate stubs do NOT signal `readyStable`, so any consumer that
488 // requires stable inputs (e.g. Tasks) waits past the precreate phase.
489 const readyStable = Object.fromEntries(
490 yield* Effect.forEach(Object.keys(allNodes), (fqn) =>
491 Effect.map(Deferred.make<void>(), (d) => [fqn, d] as const),
492 ),
493 ) as Record<string, Deferred.Deferred<void>>;
494
495 const waitForDeps = (downstreamFqn: string, fqns: string[]) =>
496 Effect.forEach(
497 fqns.filter((fqn) => fqn in ready),
498 (fqn) =>
499 // Non-cycle upstreams must be observed at their TERMINAL output
500 // (`readyStable`), not their early precreate signal (`ready`). This
501 // is what makes a failed upstream actually interrupt its downstream:
502 // a resource with `precreate` resolves `ready` before its real
503 // `reconcile` runs, so a downstream waiting on `ready` would proceed
504 // (and even finish) even though the upstream's reconcile later
505 // failed. Waiting on `readyStable` means the downstream's
506 // `waitForDeps` short-circuits with the upstream's failure cause.
507 //
508 // Cycle members are the exception: peers in an SCC depend on each
509 // other, so they must rendezvous on the early `ready`/precreate
510 // signal to break the deadlock. Phase 3 (`converge`) re-runs them
511 // against final outputs once the cycle settles. A downstream
512 // outside the cycle must still wait for reconcile: e.g. a queue
513 // consumer cannot attach to a Worker's fetch-only precreate stub.
514 plan.cycleMembers.has(downstreamFqn) && plan.cycleMembers.has(fqn)
515 ? Deferred.await(ready[fqn])
516 : Deferred.await(readyStable[fqn]),
517 { concurrency: "unbounded" },
518 );
519
520 const waitForStableDeps = (fqns: string[]) =>
521 Effect.forEach(
522 fqns.filter((fqn) => fqn in readyStable),
523 (fqn) => Deferred.await(readyStable[fqn]),
524 { concurrency: "unbounded" },
525 );
526
527 const failures: LifecycleFailure[] = [];
528
529 yield* Effect.forEach(
530 Object.entries(allNodes),
531 ([fqn, node]) =>
532 (node as ActionApply).kind === "action"
533 ? executeActionNode(
534 fqn,
535 node as ActionApply,
536 tracker,
537 outputs,
538 ready,
539 readyStable,
540 terminalStatuses,
541 reportEvent,
542 state,
543 stackName,
544 stage,
545 waitForStableDeps,
546 failures,
547 )
548 : executeNode(
549 fqn,
550 node as Apply,
551 tracker,
552 outputs,
553 ready,
554 readyStable,
555 terminalStatuses as any,
556 reportEvent,
557 state,
558 stackName,
559 stage,
560 (fqns) => waitForDeps(fqn, fqns),
561 failures,
562 plan.cycleMembers,
563 ),
564 { concurrency: "unbounded" },
565 );
566
567 if (failures.length > 0) {
568 // Aggregate every collected lifecycle failure into a single parallel Cause
569 // so the apply ends with one combined error containing every distinct
570 // failure / defect that occurred across the concurrent fibers.
571 return yield* Effect.failCause(failures.map((f) => f.cause).reduce(Cause.combine));
572 }
573 });
574
575 interface LifecycleFailure {
576 fqn: string;
577 logicalId: string;
578 type: string;
579 cause: Cause.Cause<unknown>;
580 }
581
582 /**
583 * First human-readable line of a lifecycle failure, for the per-resource
584 * `fail` status event. The full cause is still aggregated and raised when
585 * the whole apply settles — this exists so a failed row explains itself the
586 * moment it fails instead of only after every sibling has finished.
587 * `undefined` for interrupt-only causes (nothing failed, the run was
588 * cancelled).
589 */
590 const failureMessage = (cause: Cause.Cause<unknown>): string | undefined => {
591 for (const reason of cause.reasons) {
592 const error = Cause.isFailReason(reason)
593 ? reason.error
594 : Cause.isDieReason(reason)
595 ? reason.defect
596 : undefined;
597 if (error === undefined) continue;
598 const tag =
599 Predicate.hasProperty(error, "_tag") && typeof error._tag === "string"
600 ? error._tag
601 : undefined;
602 const message =
603 Predicate.hasProperty(error, "message") && typeof error.message === "string"
604 ? error.message
605 : typeof error === "string"
606 ? error
607 : Formatter.format(error);
608 const line = message.split("\n", 1)[0]!;
609 return tag !== undefined && !line.includes(tag) ? `${tag}: ${line}` : line;
610 }
611 return undefined;
612 };
613
614 const executeNode = (
615 fqn: string,
616 node: Apply,
617 tracker: Record<string, ResourceTracker>,
618 outputs: Record<string, any>,
619 ready: Record<string, Deferred.Deferred<void>>,
620 readyStable: Record<string, Deferred.Deferred<void>>,
621 terminalStatuses: Map<
622 string,
623 {
624 fqn: string;
625 id: string;
626 type: string;
627 status: Extract<ApplyStatus, "created" | "updated" | "adopted">;
628 providerMode?: ProviderMode;
629 }
630 >,
631 reportEvent: Reporter,
632 state: {
633 set: <V extends ResourceState>(req: {
634 stack: string;
635 stage: string;
636 fqn: string;
637 value: V;
638 }) => Effect.Effect<V, StateStoreError, never>;
639 },
640 stackName: string,
641 stage: string,
642 waitForDeps: (fqns: string[]) => Effect.Effect<void[], never, never>,
643 failures: LifecycleFailure[],
644 cycleMembers: ReadonlySet<string>,
645 ): Effect.Effect<void, never, never> =>
646 Effect.gen(function* () {
647 const inCycle = cycleMembers.has(fqn);
648 const logicalId = node.resource.LogicalId;
649 const namespace = node.resource.Namespace;
650
651 const commit = <S extends ResourceState>(value: Omit<S, "namespace">) =>
652 state.set({
653 stack: stackName,
654 stage,
655 fqn,
656 // Early commits (`creating`/`replacing`) persist plan props that may
657 // still hold unresolved Output exprs; strip them so state stores only
658 // plain data (see stripUnresolved in Diff.ts).
659 //
660 // Binding rows follow the same rule. Even the RESOLVED binding
661 // payload can carry Effect leaves — a tagged Worker/Function class in
662 // `env` (the circular-bindings pattern) is a function-typed Effect
663 // that `Output.evaluate` passes through untouched. A JSON state store
664 // would persist it via its `toJSON` as an `{"_id":"Effect",...}`
665 // relic, which the next plan's `diffBindings` compares against the
666 // live class stripped to `undefined` — a phantom binding "update" on
667 // every deploy, forever. Stripping at the commit boundary keeps both
668 // store kinds consistent with the comparison in `havePropsChanged`.
669 value: {
670 ...value,
671 adoptionBlocked: value.adoptionBlocked ?? node.adoptionBlocked,
672 propsIncomplete: propsIncomplete(value),
673 props: stripUnresolved(value.props),
674 bindings: stripUnresolved(value.bindings),
675 namespace,
676 } as S,
677 });
678
679 const scopedSession = resourceSession(reportEvent, fqn, logicalId);
680
681 // On a mode-switch replacement (local ⇄ live) surface the transition:
682 // the old generation's stamped mode → the mode resolved for this run.
683 const fromProviderMode =
684 node.action === "replace" &&
685 node.mode !== undefined &&
686 node.state.providerMode !== undefined &&
687 node.state.providerMode !== node.mode
688 ? node.state.providerMode
689 : undefined;
690
691 const report = (status: ApplyStatus) =>
692 reportEvent({
693 type: "apply.resource.status",
694 fqn,
695 id: logicalId,
696 resourceType: node.resource.Type,
697 status,
698 providerMode: node.mode,
699 fromProviderMode,
700 });
701
702 const markTerminal = (status: "created" | "updated") =>
703 Effect.gen(function* () {
704 terminalStatuses.set(fqn, {
705 fqn,
706 id: logicalId,
707 type: node.resource.Type,
708 status,
709 providerMode: node.mode,
710 });
711 // A local dev instance announces where it's serving: any
712 // local-mode row whose fresh Attributes carry a string `url`
713 // (Workers expose their dev-proxy URL this way) gets a
714 // `[id] ready at http://localhost:1337` line.
715 if (node.mode === "local") {
716 const url = (tracker[fqn]?.output as { url?: unknown })?.url;
717 if (typeof url === "string" && url.length > 0) {
718 yield* scopedSession.note(`ready at ${url}`);
719 }
720 }
721 // Emit immediately so the CLI surfaces the terminal status as soon
722 // as the resource is actually done — instead of batching every
723 // resource's "created"/"updated" event to the end of apply(), which
724 // makes long-running siblings appear stuck in "creating" until the
725 // entire deploy finishes.
726 //
727 // Cycle members are exempt: their initial pass produces an
728 // intermediate result that Phase 3 (`converge`) will overwrite once
729 // the SCC reaches a fixed point. Emitting "created"/"updated" here
730 // would surface that intermediate state to the CLI before the real
731 // terminal status is known. Their final status is flushed from
732 // `terminalStatuses` after `converge` completes.
733 if (inCycle) return;
734 yield* reportEvent({
735 type: "apply.resource.status",
736 fqn,
737 id: logicalId,
738 resourceType: node.resource.Type,
739 status,
740 providerMode: node.mode,
741 });
742 });
743
744 const signalReady = Deferred.succeed(ready[fqn], void 0);
745 // Signal only after reconcile completes — never during precreate. Tasks
746 // (and any other consumer that calls `waitForStableDeps`) block on this
747 // so they observe the resource's final attrs rather than a stub.
748 const signalReadyStable = Deferred.succeed(readyStable[fqn], void 0);
749
750 const storeAndSignal = (t: ResourceTracker) =>
751 Effect.gen(function* () {
752 track(tracker, outputs, fqn, t);
753 yield* signalReady;
754 });
755
756 // ── noop ──
757
758 if (node.action === "noop") {
759 // No work to do on the cloud resource — the persisted attr is already
760 // stable. Two pieces of row METADATA can still have drifted from the
761 // declaration, and this is the only pass that will ever see them:
762 //
763 // 1. `resourceType` — the row was persisted under a legacy type name
764 // (the type was since renamed and carries the old name as an
765 // alias); migrate it so the state stops depending on the alias.
766 // 2. `removalPolicy` — `RemovalPolicy.retain()` / `.destroy()` is a
767 // decoration on the declaration, not a prop, so changing it never
768 // produces a diff. Without this commit the new policy would never
769 // reach state, and the orphan delete (which reads the policy from
770 // the persisted row, see `Plan.ts`'s delete node) would act on the
771 // stale one — destroying a resource the user had marked `retain`.
772 // See https://github.com/alchemy-run/alchemy/issues/1248.
773 // 3. `downstream` — a dependent that repins from resource A to B
774 // (e.g. a HostnameAssociation switching certificates) updates
775 // *itself*, while A and B are both noops. Delete ordering reads the
776 // persisted `downstream` of the resource being deleted, so without
777 // this commit a later destroy deletes B concurrently with the
778 // dependent that still references it (and needlessly waits on A).
779 const policyChanged = node.state.removalPolicy !== node.resource.RemovalPolicy;
780 const downstreamChanged = !sameSet(node.state.downstream ?? [], node.downstream);
781 if (node.state.resourceType !== node.resource.Type || policyChanged || downstreamChanged) {
782 yield* commit({
783 ...node.state,
784 resourceType: node.resource.Type,
785 removalPolicy: node.resource.RemovalPolicy,
786 downstream: node.downstream,
787 });
788 }
789 // A policy flip is otherwise invisible (the row is a noop), and it is
790 // exactly the change a user wants confirmation of. Legacy rows with no
791 // persisted policy normalize silently — there is nothing to report.
792 if (policyChanged && node.state.removalPolicy !== undefined) {
793 yield* scopedSession.note(
794 `removal policy ${node.state.removalPolicy} → ${node.resource.RemovalPolicy}`,
795 );
796 }
797 yield* storeAndSignal({
798 output: node.state.attr,
799 props: node.state.props,
800 bindings: node.state.bindings ?? [],
801 instanceId: node.state.instanceId,
802 });
803 yield* signalReadyStable;
804 return;
805 }
806
807 const allUpstreamFqns = () => {
808 const propDeps = Object.keys(Output.resolveUpstream(node.props));
809 const bindingDeps = Object.keys(Output.resolveUpstream(node.bindings));
810 return [...new Set([...propDeps, ...bindingDeps])];
811 };
812
813 // A cycle precreates before its peers reconcile. An upstream outside the
814 // cycle cannot depend on it, so wait for those upstreams and resolve the
815 // props they determine: only values derived from cycle peers stay
816 // unresolved when precreate and `observe` run.
817 const resolvePrecreateProps = Effect.gen(function* () {
818 const outside = allUpstreamFqns().filter((upstream) => !cycleMembers.has(upstream));
819 yield* waitForDeps(outside);
820 return yield* Output.evaluateAvailable(
821 node.props,
822 Object.fromEntries(
823 outside
824 .filter((upstream) => Object.hasOwn(outputs, upstream))
825 .map((upstream) => [upstream, outputs[upstream]]),
826 ),
827 );
828 });
829
830 // ── instance ID ──
831
832 const resolveInstanceId = Effect.gen(function* () {
833 if (node.action === "create" && !node.state?.instanceId) {
834 const id = yield* generateInstanceId();
835 yield* commit<CreatingResourceState>({
836 status: "creating",
837 fqn,
838 logicalId,
839 instanceId: id,
840 downstream: node.downstream,
841 props: node.props,
842 providerVersion: node.provider.version ?? 0,
843 resourceType: node.resource.Type,
844 bindings: excludeDeletedBindings(node.bindings),
845 removalPolicy: node.resource.RemovalPolicy,
846 providerMode: node.mode,
847 adoptionBlocked: node.adoptionBlocked,
848 });
849 return id;
850 } else if (node.action === "replace") {
851 if (
852 (node.state.status === "replaced" || node.state.status === "replacing") &&
853 !node.restart
854 ) {
855 // Ordinary replacement recovery keeps using the same replacement
856 // generation. Only `restart` is allowed to mint a new instance id.
857 return node.state.instanceId;
858 }
859 const id = yield* generateInstanceId();
860 yield* commit<ReplacingResourceState>({
861 status: "replacing",
862 fqn,
863 logicalId,
864 instanceId: id,
865 downstream: node.downstream,
866 props: node.props,
867 providerVersion: node.provider.version ?? 0,
868 resourceType: node.resource.Type,
869 bindings: excludeDeletedBindings(node.bindings),
870 old: node.state,
871 deleteFirst: node.deleteFirst,
872 removalPolicy: node.resource.RemovalPolicy,
873 providerMode: node.mode,
874 });
875 return id;
876 } else if (node.state?.instanceId) {
877 return node.state.instanceId;
878 }
879 return yield* Effect.die(
880 `Instance ID not found for resource '${logicalId}' and action is '${node.action}'`,
881 );
882 });
883 const instanceId = yield* resolveInstanceId;
884
885 // ── lifecycle ──
886
887 // ── create ──
888 if (node.action === "create") {
889 if (!node.state) {
890 // First persistence point for a brand new logical resource. Once this is
891 // written, retries know they should resume creation instead of planning
892 // another fresh create from scratch.
893 yield* commit<CreatingResourceState>({
894 status: "creating",
895 fqn,
896 logicalId,
897 instanceId,
898 resourceType: node.resource.Type,
899 props: node.props,
900 attr: undefined,
901 providerVersion: node.provider.version ?? 0,
902 bindings: excludeDeletedBindings(node.bindings),
903 downstream: node.downstream,
904 removalPolicy: node.resource.RemovalPolicy,
905 providerMode: node.mode,
906 adoptionBlocked: node.adoptionBlocked,
907 });
908 }
909
910 let attr: any = node.state?.attr;
911 // Whether the identity reconcile claims has passed the ownership check.
912 let admitted = node.deferredAdoption === undefined;
913
914 if (attr !== undefined) {
915 // Precreate/read may already have produced a usable output snapshot. Publish
916 // it early so downstream resources can start resolving against it.
917 yield* storeAndSignal({ output: attr, props: {}, bindings: [], instanceId });
918 }
919
920 // Only cycle peers rendezvous on a precreate stub (`waitForDeps`); every
921 // other consumer waits for the reconciled output. Outside a cycle the
922 // stub is pure cost, and a live one (e.g. a Worker placeholder whose
923 // Durable Object classes are empty) can serve traffic while the real
924 // version propagates.
925 if (inCycle && node.provider.precreate && attr === undefined) {
926 const props = yield* resolvePrecreateProps;
927 // Without `observe`, precreate and reconcile claim the identity unchecked.
928 admitted =
929 (yield* observeBeforePrecreate(node, props, fqn, logicalId, instanceId)) !== "unresolved";
930 // Some resources need a placeholder physical resource before their real
931 // create can finish. Persist that stub so downstream evaluation can proceed.
932 yield* report("pre-creating");
933 attr = yield* node.provider
934 .precreate({
935 id: logicalId,
936 fqn,
937 news: props,
938 session: scopedSession,
939 instanceId,
940 bindings: excludeDeletedBindings(node.bindings),
941 })
942 .pipe(instrumentLifecycle("precreate", fqn, node.resource.Type, logicalId, instanceId));
943 yield* commit<CreatingResourceState>({
944 status: "creating",
945 fqn,
946 logicalId,
947 instanceId,
948 resourceType: node.resource.Type,
949 props: node.props,
950 // A stub for an unresolved identity claims nothing. Leave the
951 // checkpoint without it, so a retry checks the identity again.
952 attr: admitted ? attr : undefined,
953 providerVersion: node.provider.version ?? 0,
954 bindings: excludeDeletedBindings(node.bindings),
955 downstream: node.downstream,
956 removalPolicy: node.resource.RemovalPolicy,
957 providerMode: node.mode,
958 adoptionBlocked: node.adoptionBlocked,
959 });
960 yield* storeAndSignal({ output: attr, props: {}, bindings: [], instanceId });
961 }
962
963 // While we're waiting on upstream outputs the resource isn't actually
964 // creating anything yet — surface that as "pending" so the CLI doesn't
965 // look stuck in "creating" for slow-upstream deploys.
966 yield* report("pending");
967
968 // Create runs against fully resolved upstream outputs and bindings, not the
969 // raw Output expressions stored in the plan.
970 yield* waitForDeps(allUpstreamFqns());
971
972 yield* report("creating");
973
974 const news = (yield* Output.evaluate(node.props, outputs)) as Record<string, any>;
975
976 const bindingOutputs = excludeDeletedBindings(yield* Output.evaluate(node.bindings, outputs));
977
978 if (!admitted) {
979 const checkpoint = (attr: any) =>
980 commit<CreatingResourceState>({
981 status: "creating",
982 fqn,
983 logicalId,
984 instanceId,
985 resourceType: node.resource.Type,
986 props: news,
987 attr,
988 providerVersion: node.provider.version ?? 0,
989 bindings: bindingOutputs,
990 downstream: node.downstream,
991 removalPolicy: node.resource.RemovalPolicy,
992 providerMode: node.mode,
993 adoptionBlocked: node.adoptionBlocked,
994 });
995 // Refusal or interruption must retain identity, never foreign
996 // attributes, nor a stub for an identity not yet checked.
997 yield* checkpoint(undefined);
998 const observed = yield* observeResolved(node, news, fqn, logicalId, instanceId);
999 if (observed !== undefined) {
1000 if (Unowned.is(observed) && !node.deferredAdoption?.adopt) {
1001 return yield* new OwnedBySomeoneElse({
1002 message:
1003 `Cannot adopt resource '${fqn}' (${node.resource.Type}): ` +
1004 "it exists in the cloud but is not owned by this " +
1005 "stack/stage/logical-id. Re-run with `--adopt` (or " +
1006 "wrap the effect in `adopt(true)`) to take it over.",
1007 resourceType: node.resource.Type,
1008 logicalId,
1009 });
1010 }
1011 // Reconcile keeps a precreated stub as its output.
1012 attr ??= stripUnowned(observed);
1013 // A creating checkpoint retains olds: undefined on retry.
1014 yield* checkpoint(attr);
1015 }
1016 }
1017
1018 attr = yield* node.provider
1019 .reconcile({
1020 id: logicalId,
1021 fqn,
1022 news,
1023 instanceId,
1024 bindings: bindingOutputs,
1025 session: scopedSession,
1026 olds: undefined,
1027 output: attr,
1028 })
1029 .pipe(instrumentLifecycle("create", fqn, node.resource.Type, logicalId, instanceId));
1030
1031 yield* commit<CreatedResourceState>({
1032 status: "created",
1033 fqn,
1034 logicalId,
1035 instanceId,
1036 resourceType: node.resource.Type,
1037 props: news,
1038 attr,
1039 // Terminal commits persist the RESOLVED binding payload the
1040 // provider actually reconciled with, not the raw plan-time
1041 // expressions. Raw `node.bindings` may hold unresolved Outputs
1042 // (silently dropped by JSON state stores), so persisting them
1043 // makes the next plan's `diffBindings` compare a lossy stored
1044 // shape against fully-resolved data — a phantom binding update
1045 // on every plan (#874).
1046 bindings: bindingOutputs,
1047 providerVersion: node.provider.version ?? 0,
1048 downstream: node.downstream,
1049 removalPolicy: node.resource.RemovalPolicy,
1050 providerMode: node.mode,
1051 });
1052
1053 track(tracker, outputs, fqn, {
1054 output: attr,
1055 props: news,
1056 bindings: bindingOutputs,
1057 instanceId,
1058 });
1059 yield* signalReady;
1060 yield* signalReadyStable;
1061
1062 yield* markTerminal("created");
1063 return;
1064 }
1065
1066 // ── update ──
1067 if (node.action === "update" || node.action === "adopted") {
1068 // Cycle members publish their previous live attr *before* waiting on
1069 // upstreams so the SCC can converge — peers in the cycle would
1070 // otherwise deadlock waiting on each other. Phase 3 (`converge`)
1071 // re-runs each peer's update against fresh outputs once everyone
1072 // has settled.
1073 //
1074 // Linear (DAG) update nodes skip this entirely and simply wait for
1075 // fresh upstream outputs, mirroring the create flow. This is the
1076 // important property: a downstream of a non-cycle update never
1077 // observes the upstream's stale attr, which prevents wasted/
1078 // destructive intermediate updates (e.g. a Worker deploying with
1079 // stale Build assets).
1080 if (inCycle) {
1081 yield* storeAndSignal({
1082 output: node.state.attr,
1083 props: node.state.props,
1084 bindings: node.state.bindings ?? [],
1085 instanceId,
1086 });
1087 }
1088
1089 // See create-flow note: while we're waiting on upstream outputs
1090 // this resource isn't actually updating yet.
1091 yield* report("pending");
1092 yield* waitForDeps(allUpstreamFqns());
1093
1094 const news = (yield* Output.evaluate(node.props, outputs)) as Record<string, any>;
1095 const adopting =
1096 node.adopting === true ||
1097 (node.state.status === "updating" && node.state.adopting === true);
1098
1099 yield* node.state.status === "replaced"
1100 ? commit<ReplacedResourceState>({
1101 // Keep the replacement wrapper intact while changing the live
1102 // replacement props; GC still has older generations to delete.
1103 ...node.state,
1104 attr: node.state.attr,
1105 props: news,
1106 providerMode: node.mode,
1107 })
1108 : commit<UpdatingReourceState>({
1109 // For ordinary updates we snapshot the previously stable props/attrs
1110 // once, so retries can continue from the same baseline.
1111 status: "updating",
1112 fqn,
1113 logicalId,
1114 instanceId,
1115 resourceType: node.resource.Type,
1116 props: news,
1117 attr: node.state.attr,
1118 providerVersion: node.provider.version ?? 0,
1119 bindings: excludeDeletedBindings(node.bindings),
1120 downstream: node.downstream,
1121 old: node.state.status === "updating" ? node.state.old : node.state,
1122 adopting: adopting ? true : undefined,
1123 removalPolicy: node.resource.RemovalPolicy,
1124 providerMode: node.mode,
1125 });
1126
1127 yield* report(node.action === "adopted" ? "adopting" : "updating");
1128
1129 const previousProps = adopting
1130 ? undefined
1131 : node.state.status === "created" ||
1132 node.state.status === "updated" ||
1133 node.state.status === "replaced"
1134 ? node.state.props
1135 : node.state.old.props;
1136
1137 // Providers receive the resolved binding payload for this exact pass, while
1138 // `previousProps` tells them what state the live resource is being updated from.
1139 const bindingOutputs = excludeDeletedBindings(yield* Output.evaluate(node.bindings, outputs));
1140
1141 const attr = yield* node.provider
1142 .reconcile({
1143 id: logicalId,
1144 fqn,
1145 news,
1146 instanceId,
1147 bindings: bindingOutputs,
1148 session: scopedSession,
1149 olds: previousProps,
1150 output: node.state.attr,
1151 })
1152 .pipe(instrumentLifecycle("update", fqn, node.resource.Type, logicalId, instanceId));
1153
1154 if (node.state.status === "replaced") {
1155 yield* commit<ReplacedResourceState>({
1156 // The live replacement changed, but cleanup of older generations still
1157 // has to continue afterwards.
1158 ...node.state,
1159 attr,
1160 props: news,
1161 // Resolved payload, not raw `node.bindings` — see create commit.
1162 bindings: bindingOutputs,
1163 providerMode: node.mode,
1164 });
1165 } else {
1166 yield* commit<UpdatedResourceState>({
1167 status: "updated",
1168 fqn,
1169 logicalId,
1170 instanceId,
1171 resourceType: node.resource.Type,
1172 props: news,
1173 attr,
1174 // Resolved payload, not raw `node.bindings` — see create commit.
1175 bindings: bindingOutputs,
1176 providerVersion: node.provider.version ?? 0,
1177 downstream: node.downstream,
1178 removalPolicy: node.resource.RemovalPolicy,
1179 providerMode: node.mode,
1180 });
1181 }
1182
1183 track(tracker, outputs, fqn, {
1184 output: attr,
1185 props: news,
1186 bindings: bindingOutputs,
1187 instanceId,
1188 });
1189 // Signal here for the linear (non-cycle) path. For in-cycle updates
1190 // the deferred has already been resolved by the early `storeAndSignal`
1191 // above and `signalReady` is a no-op the second time.
1192 yield* signalReady;
1193 yield* signalReadyStable;
1194
1195 if (node.action === "adopted") {
1196 terminalStatuses.set(fqn, {
1197 fqn,
1198 id: logicalId,
1199 type: node.resource.Type,
1200 status: "adopted",
1201 providerMode: node.mode,
1202 });
1203 if (!inCycle) yield* report("adopted");
1204 } else {
1205 yield* markTerminal("updated");
1206 }
1207 return;
1208 }
1209
1210 // ── replace ──
1211 if (node.action === "replace") {
1212 if (node.state.status === "replaced" && !node.restart) {
1213 // The replacement already exists; this pass only needs GC to clean up
1214 // older generations, so expose the current replacement and stop here.
1215 track(tracker, outputs, fqn, {
1216 output: node.state.attr,
1217 props: node.state.props,
1218 bindings: node.state.bindings ?? [],
1219 instanceId,
1220 });
1221 yield* signalReady;
1222 yield* signalReadyStable;
1223 yield* markTerminal("created");
1224 return;
1225 }
1226
1227 let replState: ReplacingResourceState;
1228 if (node.state.status !== "replacing" || node.restart) {
1229 // `restart` deliberately nests the previous top-level replacement state
1230 // into `old`, creating a new outer generation to replace it.
1231 replState = yield* commit<ReplacingResourceState>({
1232 status: "replacing",
1233 fqn,
1234 logicalId,
1235 instanceId,
1236 resourceType: node.resource.Type,
1237 props: node.props,
1238 bindings: excludeDeletedBindings(node.bindings),
1239 attr: undefined,
1240 providerVersion: node.provider.version ?? 0,
1241 deleteFirst: node.deleteFirst,
1242 old: node.state,
1243 downstream: node.downstream,
1244 removalPolicy: node.resource.RemovalPolicy,
1245 providerMode: node.mode,
1246 });
1247 } else {
1248 // Resume the same replacement generation after an interrupted apply.
1249 replState = node.state;
1250 }
1251
1252 // ── delete-first replacements ──
1253 //
1254 // By default a replacement is create-first: the new generation is
1255 // created here and the old generation(s) are reclaimed afterwards by
1256 // `collectGarbage` (Phase 2). That ordering keeps the old resource
1257 // alive if the create fails, but it is wrong for resources whose
1258 // replacement cannot coexist with the original — a fixed physical
1259 // name, a singleton, etc. Those providers return
1260 // `{ action: "replace", deleteFirst: true }`.
1261 //
1262 // When `deleteFirst` is set we tear the previous generation(s) down
1263 // BEFORE creating the new one, and commit the result as a terminal
1264 // `created` state (rather than `replaced`) so Phase 2 has no old chain
1265 // left to drain. `delete` is required to be idempotent, so a re-run
1266 // after an interrupted apply simply re-converges.
1267 const deleteOldGenerations = (
1268 old: ReplacementOldResourceState,
1269 ): Effect.Effect<void, any, any> =>
1270 Effect.gen(function* () {
1271 const retain = node.resource.RemovalPolicy === "retain";
1272 if (old.attr !== undefined && !retain) {
1273 // Delete each old generation with the provider variant of the
1274 // mode that created it — after a local ⇄ live switch,
1275 // `node.provider` (the new mode) cannot tear down the other
1276 // runtime's instance. Unstamped rows (legacy or written by a
1277 // mode-agnostic provider) are physically live, unless their
1278 // attrs carry the `dev:` identity marker — see stampedMode.
1279 const oldProvider = yield* findProviderByType(node.resource.Type, stampedMode(old));
1280 yield* oldProvider
1281 .delete({
1282 id: logicalId,
1283 fqn,
1284 instanceId: old.instanceId,
1285 olds: old.props as never,
1286 output: old.attr,
1287 session: scopedSession,
1288 bindings: [],
1289 })
1290 .pipe(
1291 instrumentLifecycle("delete", fqn, node.resource.Type, logicalId, old.instanceId),
1292 );
1293 }
1294 if (old.status === "replacing" || old.status === "replaced") {
1295 yield* deleteOldGenerations(old.old);
1296 }
1297 });
1298
1299 const deleteFirst = Effect.fn(function* () {
1300 yield* scopedSession.note(
1301 "Deleting previous resource before creating its replacement (deleteFirst)...",
1302 );
1303 yield* deleteOldGenerations(replState.old);
1304 });
1305
1306 let attr: any = replState.attr;
1307
1308 // ── replacement admission ──
1309 //
1310 // The replacement may resolve to a physical identity that already
1311 // exists and belongs to someone else (e.g. a renamed Worker pointing at
1312 // a foreign script). Read it before anything is mutated — before
1313 // precreate, reconcile, and delete-first cleanup of the previous
1314 // generation — and refuse an unowned target unless adoption is allowed,
1315 // exactly like a fresh create.
1316 let admitted = attr !== undefined || node.deferredAdoption === undefined;
1317 const admit = Effect.fn(function* (props: Record<string, any>) {
1318 admitted = true;
1319 const observed = yield* observeResolved(
1320 node,
1321 props,
1322 fqn,
1323 logicalId,
1324 instanceId,
1325 replState.old?.attr,
1326 );
1327 if (observed === undefined) return;
1328 if (Unowned.is(observed) && !node.deferredAdoption?.adopt) {
1329 return yield* new OwnedBySomeoneElse({
1330 message:
1331 `Cannot replace resource '${fqn}' (${node.resource.Type}): ` +
1332 "its replacement exists in the cloud but is not owned by this " +
1333 "stack/stage/logical-id. Re-run with `--adopt` (or " +
1334 "wrap the effect in `adopt(true)`) to take it over.",
1335 resourceType: node.resource.Type,
1336 logicalId,
1337 });
1338 }
1339 // A delete-first target may be the previous generation itself, which
1340 // is about to be deleted — never hand its attributes to reconcile.
1341 // Reconcile keeps a precreated stub as its output.
1342 if (!node.deleteFirst) attr ??= stripUnowned(observed);
1343 });
1344
1345 // A cycle precreates on the props its upstreams outside the cycle
1346 // determine; anything else waits for every upstream.
1347 const precreateProps = inCycle ? yield* resolvePrecreateProps : node.props;
1348
1349 if (!admitted && isResolved(precreateProps)) {
1350 yield* admit(precreateProps);
1351 }
1352
1353 // A cycle precreates before its props resolve, so `observe` admits the
1354 // identity precreate will claim — before delete-first, too. Without
1355 // `observe`, precreate and reconcile claim it unchecked. An unresolved
1356 // identity is admitted once it resolves, before reconcile.
1357 if (!admitted && inCycle) {
1358 const observation = yield* observeBeforePrecreate(
1359 node,
1360 precreateProps,
1361 fqn,
1362 logicalId,
1363 instanceId,
1364 replState.old?.attr,
1365 );
1366 admitted = observation !== "unresolved";
1367 if (!admitted && node.deleteFirst && !node.deferredAdoption?.adopt) {
1368 return yield* new UnverifiableReplacement({
1369 message:
1370 `Cannot replace resource '${fqn}' (${node.resource.Type}): ` +
1371 "it must be deleted before its replacement is pre-created, but " +
1372 "the replacement's identity depends on outputs that are not " +
1373 "resolved yet, so its ownership cannot be checked first. Re-run " +
1374 "with `--adopt` (or wrap the effect in `adopt(true)`) to replace it anyway.",
1375 resourceType: node.resource.Type,
1376 logicalId,
1377 });
1378 }
1379 }
1380
1381 // A cycle must delete before precreate claims the identity. Plan has
1382 // already refused one whose identity could not be admitted first
1383 // without `observe`; the check above refuses an unresolved one.
1384 // Otherwise the previous generation survives until reconcile is
1385 // about to run, so a failed upstream never leaves the resource missing.
1386 let deletedFirst = false;
1387 if (node.deleteFirst && inCycle) {
1388 yield* deleteFirst();
1389 deletedFirst = true;
1390 }
1391
1392 if (attr !== undefined) {
1393 // If precreate already ran, expose that intermediate output immediately so
1394 // downstream resources can resolve against the same in-flight replacement.
1395 yield* storeAndSignal({ output: attr, props: {}, bindings: [], instanceId });
1396 }
1397
1398 if (inCycle && node.provider.precreate && attr === undefined) {
1399 yield* report("pre-creating");
1400 attr = yield* node.provider
1401 .precreate({
1402 id: logicalId,
1403 fqn,
1404 news: precreateProps,
1405 session: scopedSession,
1406 instanceId,
1407 bindings: excludeDeletedBindings(node.bindings),
1408 })
1409 .pipe(instrumentLifecycle("precreate", fqn, node.resource.Type, logicalId, instanceId));
1410 yield* commit<ReplacingResourceState>({
1411 status: "replacing",
1412 fqn,
1413 logicalId,
1414 instanceId,
1415 resourceType: node.resource.Type,
1416 props: node.props,
1417 // A stub for an unresolved identity claims nothing. Leave the
1418 // checkpoint without it, so a retry checks the identity again.
1419 attr: admitted ? attr : undefined,
1420 providerVersion: node.provider.version ?? 0,
1421 bindings: excludeDeletedBindings(node.bindings),
1422 downstream: node.downstream,
1423 old: replState.old,
1424 deleteFirst: node.deleteFirst,
1425 removalPolicy: node.resource.RemovalPolicy,
1426 providerMode: node.mode,
1427 });
1428 yield* storeAndSignal({ output: attr, props: {}, bindings: [], instanceId });
1429 }
1430
1431 // See create-flow note: while we're waiting on upstream outputs
1432 // the replacement isn't actually being created yet.
1433 yield* report("pending");
1434
1435 // Replacement create is evaluated exactly like create, but against the new
1436 // generation's instance id and with the previous generations preserved in `old`.
1437 yield* waitForDeps(allUpstreamFqns());
1438
1439 yield* report("creating replacement");
1440
1441 const news = (yield* Output.evaluate(node.props, outputs)) as Record<string, any>;
1442
1443 const bindingOutputs = excludeDeletedBindings(yield* Output.evaluate(node.bindings, outputs));
1444
1445 if (!admitted) {
1446 yield* admit(news);
1447 }
1448 if (node.deleteFirst && !deletedFirst) {
1449 yield* deleteFirst();
1450 }
1451
1452 attr = yield* node.provider
1453 .reconcile({
1454 id: logicalId,
1455 fqn,
1456 news,
1457 instanceId,
1458 bindings: bindingOutputs,
1459 session: scopedSession,
1460 olds: undefined,
1461 output: attr,
1462 })
1463 .pipe(instrumentLifecycle("create", fqn, node.resource.Type, logicalId, instanceId));
1464
1465 if (node.deleteFirst) {
1466 // The old generation(s) were already torn down above, so there is
1467 // nothing left for `collectGarbage` to drain — collapse straight to
1468 // the terminal `created` state.
1469 yield* commit<CreatedResourceState>({
1470 status: "created",
1471 fqn,
1472 logicalId,
1473 instanceId,
1474 resourceType: node.resource.Type,
1475 props: news,
1476 attr,
1477 providerVersion: node.provider.version ?? 0,
1478 // Resolved payload, not raw `node.bindings` — see create commit.
1479 bindings: bindingOutputs,
1480 downstream: node.downstream,
1481 removalPolicy: node.resource.RemovalPolicy,
1482 providerMode: node.mode,
1483 });
1484 } else {
1485 yield* commit<ReplacedResourceState>({
1486 // Creation of the new generation succeeded; from here on the only remaining
1487 // work is draining the old chain via garbage collection.
1488 status: "replaced",
1489 fqn,
1490 logicalId,
1491 instanceId,
1492 resourceType: node.resource.Type,
1493 props: news,
1494 attr,
1495 providerVersion: node.provider.version ?? 0,
1496 // Resolved payload, not raw `node.bindings` — see create commit.
1497 bindings: bindingOutputs,
1498 downstream: node.downstream,
1499 // Preserve the remaining backlog exactly as-is. GC is responsible for
1500 // popping one generation at a time until the chain is exhausted.
1501 old: replState.old,
1502 deleteFirst: node.deleteFirst,
1503 removalPolicy: node.resource.RemovalPolicy,
1504 providerMode: node.mode,
1505 });
1506 }
1507
1508 track(tracker, outputs, fqn, {
1509 output: attr,
1510 props: news,
1511 bindings: bindingOutputs,
1512 instanceId,
1513 });
1514 yield* signalReady;
1515 yield* signalReadyStable;
1516
1517 // Keep progress anchored to the live replacement while GC drains the
1518 // previous generation(s) in the background.
1519 yield* markTerminal("created");
1520 return;
1521 }
1522
1523 return yield* Effect.die(`Unknown action: ${(node as { action: string }).action}`);
1524 }).pipe(
1525 Effect.catchCause((cause) =>
1526 // Record the failure, propagate it to any downstream resources waiting on
1527 // our Deferred (so their waitForDeps short-circuits instead of deadlocking),
1528 // emit a "fail" status, and resolve to void so Effect.all
1529 // does not interrupt sibling fibers. The aggregated cause is raised at
1530 // the end of executePlan.
1531 Effect.gen(function* () {
1532 failures.push({ fqn, logicalId: node.resource.LogicalId, type: node.resource.Type, cause });
1533 yield* Deferred.failCause(ready[fqn], cause as Cause.Cause<never>);
1534 yield* Deferred.failCause(readyStable[fqn], cause as Cause.Cause<never>);
1535 yield* reportEvent({
1536 type: "apply.resource.status",
1537 fqn,
1538 id: node.resource.LogicalId,
1539 resourceType: node.resource.Type,
1540 status: "fail",
1541 message: failureMessage(cause),
1542 providerMode: node.mode,
1543 });
1544 }),
1545 ),
1546 Effect.withSpan("apply.resource", {
1547 attributes: {
1548 "alchemy.resource.fqn": fqn,
1549 "alchemy.resource.type": node.resource.Type,
1550 "alchemy.resource.logical_id": node.resource.LogicalId,
1551 "alchemy.resource.action": node.action,
1552 },
1553 }),
1554 ) as Effect.Effect<void, never, never>;
1555
1556 // ── Task execution ─────────────────────────────────────────────────────────
1557 //
1558 // Tasks slot into the same scheduler as resources. They have no provider
1559 // lifecycle — just a single Effect that runs when inputs change (or when
1560 // `--force` is set). The output value is written to `tracker[fqn].output`
1561 // so downstream Output evaluation works identically to resource attrs.
1562
1563 /**
1564 * Run an Action body, resolving any Outputs it captured via `yield* output`
1565 * during init against the current tracker and exposing them to the body through
1566 * the resolve {@link RuntimeContext}. See {@link makeCaptureContext}.
1567 */
1568 const runAction = Effect.fn("apply.runAction")(function* (
1569 task: ActionLike,
1570 input: any,
1571 outputs: Record<string, any>,
1572 ) {
1573 const resolved: Record<string, unknown> = {};
1574 for (const [key, output] of Object.entries(task.Captures)) {
1575 resolved[key] = yield* Output.evaluate(output, outputs);
1576 }
1577 return yield* task
1578 .Run(input)
1579 .pipe(Effect.provideService(RuntimeContext, makeResolveContext(resolved)));
1580 });
1581
1582 const executeActionNode = (
1583 fqn: string,
1584 node: ActionApply,
1585 tracker: Record<string, ResourceTracker>,
1586 outputs: Record<string, any>,
1587 ready: Record<string, Deferred.Deferred<void>>,
1588 readyStable: Record<string, Deferred.Deferred<void>>,
1589 terminalStatuses: Map<
1590 string,
1591 {
1592 fqn: string;
1593 id: string;
1594 type: string;
1595 status: Extract<ApplyStatus, "created" | "updated" | "adopted" | "ran" | "skipped">;
1596 providerMode?: ProviderMode;
1597 }
1598 >,
1599 reportEvent: Reporter,
1600 state: {
1601 set: <V extends PersistedState>(req: {
1602 stack: string;
1603 stage: string;
1604 fqn: string;
1605 value: V;
1606 }) => Effect.Effect<V, StateStoreError, never>;
1607 },
1608 stackName: string,
1609 stage: string,
1610 waitForDeps: (fqns: string[]) => Effect.Effect<void[], never, never>,
1611 failures: LifecycleFailure[],
1612 ): Effect.Effect<void, never, any> =>
1613 Effect.gen(function* () {
1614 const task = node.def;
1615 const logicalId = task.LogicalId;
1616 const namespace = task.Namespace;
1617
1618 const commit = <S extends ActionState>(value: Omit<S, "namespace">) =>
1619 state.set({ stack: stackName, stage, fqn, value: { ...value, namespace } as S });
1620
1621 const report = (status: ApplyStatus) =>
1622 reportEvent({
1623 type: "apply.resource.status",
1624 fqn,
1625 id: logicalId,
1626 resourceType: task.Type,
1627 status,
1628 });
1629
1630 const signalReady = Deferred.succeed(ready[fqn], void 0);
1631 const signalReadyStable = Deferred.succeed(readyStable[fqn], void 0);
1632
1633 const skip = (state: RanActionState) =>
1634 Effect.gen(function* () {
1635 if (!sameSet(state.downstream ?? [], node.downstream)) {
1636 yield* commit<RanActionState>({ ...state, downstream: node.downstream });
1637 }
1638 track(tracker, outputs, fqn, {
1639 output: state.output,
1640 props: { __input: state.input },
1641 bindings: [],
1642 instanceId: fqn,
1643 });
1644 yield* signalReady;
1645 yield* signalReadyStable;
1646 terminalStatuses.set(fqn, { fqn, id: logicalId, type: task.Type, status: "skipped" });
1647 yield* report("skipped");
1648 });
1649
1650 if (node.action === "noop") {
1651 return yield* skip(node.state);
1652 }
1653
1654 // ── run ──
1655 // Tasks wait on `waitForStableDeps` (post-reconcile attrs) for their
1656 // upstreams, which is often the slowest dep chain in the deploy.
1657 // Surface that as "pending" instead of having the task show no status
1658 // until its run actually starts.
1659 yield* report("pending");
1660 yield* waitForDeps(
1661 [
1662 ...new Set([
1663 ...Object.keys(Output.resolveUpstream(node.input)),
1664 ...Object.keys(Output.upstreamAny(task.Captures)),
1665 ]),
1666 ].filter((f) => f in readyStable),
1667 );
1668
1669 const resolvedInput = (yield* Output.evaluate(node.input, outputs)) as any;
1670 const inputHashValue = yield* hashInput(resolvedInput);
1671
1672 // Inputs unknown during planning may resolve to the last successful input.
1673 if (!node.forced && node.state?.status === "ran" && node.state.inputHash === inputHashValue) {
1674 return yield* skip(node.state);
1675 }
1676
1677 yield* commit<RunningActionState>({
1678 kind: "action",
1679 status: "running",
1680 fqn,
1681 logicalId,
1682 actionType: task.Type,
1683 inputHash: inputHashValue,
1684 input: resolvedInput,
1685 downstream: node.downstream,
1686 });
1687 yield* report("running");
1688
1689 const result = yield* runAction(task, resolvedInput, outputs);
1690
1691 yield* commit<RanActionState>({
1692 kind: "action",
1693 status: "ran",
1694 fqn,
1695 logicalId,
1696 actionType: task.Type,
1697 inputHash: inputHashValue,
1698 input: resolvedInput,
1699 output: result,
1700 downstream: node.downstream,
1701 });
1702
1703 track(tracker, outputs, fqn, {
1704 output: result,
1705 props: { __input: resolvedInput },
1706 bindings: [],
1707 instanceId: fqn,
1708 });
1709 yield* signalReady;
1710 yield* signalReadyStable;
1711 terminalStatuses.set(fqn, { fqn, id: logicalId, type: task.Type, status: "ran" });
1712 yield* report("ran");
1713 }).pipe(
1714 Effect.catchCause((cause) =>
1715 Effect.gen(function* () {
1716 failures.push({ fqn, logicalId: node.def.LogicalId, type: node.def.Type, cause });
1717 yield* Deferred.failCause(ready[fqn], cause as Cause.Cause<never>);
1718 yield* Deferred.failCause(readyStable[fqn], cause as Cause.Cause<never>);
1719 yield* reportEvent({
1720 type: "apply.resource.status",
1721 fqn,
1722 id: node.def.LogicalId,
1723 resourceType: node.def.Type,
1724 status: "fail",
1725 message: failureMessage(cause),
1726 });
1727 }),
1728 ),
1729 Effect.withSpan("apply.action", {
1730 attributes: {
1731 "alchemy.action.fqn": fqn,
1732 "alchemy.action.type": node.def.Type,
1733 "alchemy.action.logical_id": node.def.LogicalId,
1734 "alchemy.action.verb": node.action,
1735 },
1736 }),
1737 ) as Effect.Effect<void, never, any>;
1738
1739 // ── Phase 3: imperative convergence loop ───────────────────────────────────
1740 //
1741 // After the initial concurrent pass, some resources may have been created
1742 // with stale upstream values (e.g. a precreate stub instead of the final
1743 // output). Walk the plan and re-evaluate each resource's props/bindings
1744 // against the current tracker outputs. Call provider.reconcile for any
1745 // resource whose resolved inputs differ from what it was last applied with.
1746 // Repeat until no resource needs updating.
1747
1748 const converge = Effect.fn(function* (
1749 plan: Plan,
1750 tracker: Record<string, ResourceTracker>,
1751 outputs: Record<string, any>,
1752 terminalStatuses: Map<
1753 string,
1754 {
1755 fqn: string;
1756 id: string;
1757 type: string;
1758 status: Extract<ApplyStatus, "created" | "updated" | "adopted" | "ran" | "skipped">;
1759 providerMode?: ProviderMode;
1760 }
1761 >,
1762 reportEvent: Reporter,
1763 state: {
1764 set: <V extends PersistedState>(req: {
1765 stack: string;
1766 stage: string;
1767 fqn: string;
1768 value: V;
1769 }) => Effect.Effect<V, StateStoreError, never>;
1770 },
1771 stackName: string,
1772 stage: string,
1773 ) {
1774 for (;;) {
1775 let anyUpdated = false;
1776
1777 for (const [fqn, node] of Object.entries(plan.resources)) {
1778 if (node.action === "noop") continue;
1779 if (!tracker[fqn]) continue;
1780
1781 const newProps = (yield* Output.evaluate(node.props, outputs)) as Record<string, any>;
1782
1783 const newBindings = excludeDeletedBindings(yield* Output.evaluate(node.bindings, outputs));
1784
1785 const oldProps = tracker[fqn].props;
1786 const oldBindings = tracker[fqn].bindings;
1787
1788 const propsChanged = havePropsChanged(oldProps, newProps);
1789 const bindingsChanged = JSON.stringify(oldBindings) !== JSON.stringify(newBindings);
1790
1791 if (!propsChanged && !bindingsChanged) continue;
1792
1793 anyUpdated = true;
1794
1795 const logicalId = node.resource.LogicalId;
1796 const namespace = node.resource.Namespace;
1797 const instanceId = tracker[fqn].instanceId;
1798
1799 const scopedSession = resourceSession(reportEvent, fqn, logicalId);
1800
1801 // A cycle peer resolves against its peers' first outputs, so an
1802 // identity checked then may resolve differently now. Check the identity
1803 // these props resolve to before reconcile claims it.
1804 const deferredAdoption = "deferredAdoption" in node ? node.deferredAdoption : undefined;
1805 if (propsChanged && deferredAdoption !== undefined) {
1806 const observed = yield* observeResolved(
1807 node,
1808 newProps,
1809 fqn,
1810 logicalId,
1811 instanceId,
1812 tracker[fqn].output,
1813 );
1814 if (observed !== undefined && Unowned.is(observed) && !deferredAdoption.adopt) {
1815 return yield* new OwnedBySomeoneElse({
1816 message:
1817 `Cannot converge resource '${fqn}' (${node.resource.Type}): ` +
1818 "its final identity exists in the cloud but is not owned by this " +
1819 "stack/stage/logical-id. Re-run with `--adopt` (or " +
1820 "wrap the effect in `adopt(true)`) to take it over.",
1821 resourceType: node.resource.Type,
1822 logicalId,
1823 });
1824 }
1825 }
1826
1827 const attr = yield* node.provider
1828 .reconcile({
1829 id: logicalId,
1830 fqn,
1831 news: newProps,
1832 instanceId,
1833 bindings: newBindings,
1834 session: scopedSession,
1835 olds: oldProps,
1836 output: tracker[fqn].output,
1837 })
1838 .pipe(instrumentLifecycle("update", fqn, node.resource.Type, logicalId, instanceId));
1839
1840 track(tracker, outputs, fqn, {
1841 output: attr,
1842 props: newProps,
1843 bindings: newBindings,
1844 instanceId,
1845 });
1846
1847 yield* state.set({
1848 stack: stackName,
1849 stage,
1850 fqn,
1851 value: {
1852 status: "updated",
1853 fqn,
1854 logicalId,
1855 instanceId,
1856 resourceType: node.resource.Type,
1857 // This site bypasses the `commit` helper, so strip unresolved
1858 // leaves (Effect-valued env entries survive `Output.evaluate`)
1859 // the same way `commit` does — state only ever holds plain data.
1860 props: stripUnresolved(newProps),
1861 attr,
1862 providerVersion: node.provider.version ?? 0,
1863 // Resolved payload, not raw `node.bindings` — see the create
1864 // commit in applyResource. Stripped like the commit helper so
1865 // Effect leaves (e.g. tagged Worker classes in `env`) never reach
1866 // the state store.
1867 bindings: stripUnresolved(newBindings),
1868 downstream: node.downstream,
1869 namespace,
1870 removalPolicy: node.resource.RemovalPolicy,
1871 providerMode: node.mode,
1872 adoptionBlocked: node.adoptionBlocked,
1873 } as UpdatedResourceState,
1874 });
1875
1876 terminalStatuses.set(fqn, {
1877 fqn,
1878 id: logicalId,
1879 type: node.resource.Type,
1880 status: node.action === "adopted" ? "adopted" : "updated",
1881 providerMode: node.mode,
1882 });
1883 }
1884
1885 // Tasks: re-run when their resolved input drifts vs. the value they
1886 // last applied with (e.g. an upstream resource produced new attrs in
1887 // this pass). Skipped (noop) tasks are not re-checked here — their
1888 // recorded inputHash is authoritative until the next plan.
1889 for (const [fqn, node] of Object.entries(plan.actions)) {
1890 if (node.action !== "run") continue;
1891 if (!tracker[fqn]) continue;
1892
1893 const newInput = (yield* Output.evaluate(node.input, outputs)) as any;
1894 const newHash = yield* hashInput(newInput);
1895 const oldInput = tracker[fqn].props?.__input;
1896 const oldHash = yield* hashInput(oldInput);
1897 if (newHash === oldHash) continue;
1898
1899 anyUpdated = true;
1900
1901 yield* state.set({
1902 stack: stackName,
1903 stage,
1904 fqn,
1905 value: {
1906 kind: "action",
1907 status: "running",
1908 fqn,
1909 logicalId: node.def.LogicalId,
1910 namespace: node.def.Namespace,
1911 actionType: node.def.Type,
1912 inputHash: newHash,
1913 input: newInput,
1914 downstream: node.downstream,
1915 } satisfies RunningActionState,
1916 });
1917
1918 const result = yield* runAction(node.def, newInput, outputs);
1919
1920 yield* state.set({
1921 stack: stackName,
1922 stage,
1923 fqn,
1924 value: {
1925 kind: "action",
1926 status: "ran",
1927 fqn,
1928 logicalId: node.def.LogicalId,
1929 namespace: node.def.Namespace,
1930 actionType: node.def.Type,
1931 inputHash: newHash,
1932 input: newInput,
1933 output: result,
1934 downstream: node.downstream,
1935 } satisfies RanActionState,
1936 });
1937
1938 track(tracker, outputs, fqn, {
1939 output: result,
1940 props: { __input: newInput },
1941 bindings: [],
1942 instanceId: fqn,
1943 });
1944 terminalStatuses.set(fqn, {
1945 fqn,
1946 id: node.def.LogicalId,
1947 type: node.def.Type,
1948 status: "ran",
1949 });
1950 }
1951
1952 if (!anyUpdated) break;
1953 }
1954 });
1955
1956 // ── Phase 2: delete orphans and old replaced resources ─────────────────────
1957
1958 /** A provider delete (or its attr-recovery read / state commit) that failed. */
1959 export interface DeleteFailure {
1960 fqn: string;
1961 logicalId: string;
1962 resourceType: string;
1963 cause: Cause.Cause<unknown>;
1964 }
1965
1966 /**
1967 * A delete that was never attempted because a dependent's delete failed (or
1968 * was itself blocked). The resource may legitimately be undeletable while its
1969 * dependents still exist, so skipping is not an error in its own right.
1970 */
1971 export interface BlockedDelete {
1972 fqn: string;
1973 logicalId: string;
1974 resourceType: string;
1975 /** FQNs of the dependents whose failed/blocked deletes block this one. */
1976 blockedBy: string[];
1977 }
1978
1979 /**
1980 * Aggregate raised at the end of the deletion phase when one or more
1981 * resource deletes failed. Every resource whose delete did not depend on a
1982 * failed one was still attempted — a single failure no longer strands
1983 * unrelated siblings.
1984 */
1985 export class DestroyError extends Data.TaggedError("DestroyError")<{
1986 failures: ReadonlyArray<DeleteFailure>;
1987 blocked: ReadonlyArray<BlockedDelete>;
1988 }> {
1989 override get message(): string {
1990 return [
1991 `Failed to delete ${this.failures.length} resource(s)` +
1992 (this.blocked.length > 0
1993 ? ` (${this.blocked.length} more skipped because a dependent's delete failed)`
1994 : "") +
1995 ":",
1996 ...this.failures.map((f) => ` ✗ ${f.fqn} (${f.resourceType}): ${Cause.pretty(f.cause)}`),
1997 ...this.blocked.map(
1998 (b) =>
1999 ` ⊘ ${b.fqn} (${b.resourceType}): skipped — blocked by failed delete of ${b.blockedBy.join(", ")}`,
2000 ),
2001 ].join("\n");
2002 }
2003 }
2004
2005 const collectGarbage = Effect.fn(function* (plan: Plan, reportEvent: Reporter) {
2006 const state = yield* yield* State;
2007 const stack = yield* Stack;
2008 const stackName = stack.name;
2009 const stage = yield* Stage;
2010
2011 // Task deletions are pure state drops — no body is invoked. Run them in
2012 // parallel before resource GC; tasks never have provider-side dependencies
2013 // to wait on.
2014 yield* Effect.forEach(
2015 Object.entries(plan.actionDeletions ?? {}),
2016 ([fqn, node]) =>
2017 node === undefined
2018 ? Effect.void
2019 : Effect.gen(function* () {
2020 yield* reportEvent({
2021 type: "apply.resource.status",
2022 fqn,
2023 id: node.def.LogicalId,
2024 resourceType: node.def.Type,
2025 status: "deleting",
2026 });
2027 yield* state.delete({ stack: stackName, stage, fqn });
2028 yield* reportEvent({
2029 type: "apply.resource.status",
2030 fqn,
2031 id: node.def.LogicalId,
2032 resourceType: node.def.Type,
2033 status: "deleted",
2034 });
2035 }),
2036 { concurrency: "unbounded" },
2037 );
2038
2039 // Failures are collected — not propagated — so one bad delete never
2040 // strands unrelated siblings. `unresolved` tracks every FQN whose delete
2041 // failed or was blocked this run: later passes must not retry them (a
2042 // still-`replaced` row would otherwise spin the drain loop forever) and
2043 // dependencies scheduled in later passes must observe them as blocking.
2044 const failures: DeleteFailure[] = [];
2045 const blockedDeletes: BlockedDelete[] = [];
2046 const unresolved = new Set<string>();
2047 // Skipped by later passes so generations older than a draining one are still reclaimed.
2048 const inProgress = new Set<string>();
2049
2050 const pendingDeletes = { ...plan.deletions };
2051 // Pending means this generation drained; deferred means it is still live.
2052 type DeleteOutcome = "deleted" | "pending" | "deferred" | "failed" | "blocked";
2053
2054 const deleteGraph = Effect.fn(function* (
2055 deletionGraph: Record<string, Delete | ReplacementResourceState | undefined>,
2056 ) {
2057 const deletions: { [fqn: string]: Effect.Effect<DeleteOutcome, never, ArtifactStore> } = {};
2058
2059 const deleteResource = (
2060 node: Delete | ReplacementResourceState,
2061 ancestors: ReadonlySet<string> = new Set(),
2062 ): Effect.Effect<DeleteOutcome, never, ArtifactStore> =>
2063 Effect.gen(function* () {
2064 const isDeleteNode = (node: Delete | ReplacementResourceState): node is Delete =>
2065 "action" in node;
2066
2067 const generation = isDeleteNode(node)
2068 ? node.state
2069 : (nextOldGeneration(node, inProgress) ?? node.old);
2070
2071 const {
2072 fqn,
2073 logicalId,
2074 namespace,
2075 resourceType,
2076 instanceId,
2077 downstream,
2078 props,
2079 attr: persistedAttr,
2080 provider,
2081 providerMode,
2082 } = isDeleteNode(node)
2083 ? {
2084 // Use the persisted FQN verbatim — never recompute it from
2085 // `toFqn(namespace, logicalId)`. A logical ID may legitimately
2086 // contain the FQN separator (`/`), in which case `parseFqn`
2087 // truncated it and a recomputed key would miss the real state
2088 // row (the row would then resurface as an orphan on every
2089 // subsequent destroy, never getting deleted).
2090 fqn: node.resource.FQN,
2091 logicalId: node.resource.LogicalId,
2092 namespace: node.resource.Namespace,
2093 resourceType: node.resource.Type,
2094 instanceId: node.state.instanceId,
2095 downstream: node.downstream,
2096 props: node.state.props,
2097 attr: node.state.attr,
2098 // Plan resolved this provider for the row's persisted (or
2099 // marker-inferred) `providerMode` (see the deletions builder
2100 // in Plan.ts).
2101 provider: node.provider,
2102 providerMode: node.state.providerMode,
2103 }
2104 : {
2105 fqn: node.fqn,
2106 logicalId: node.logicalId,
2107 namespace: node.namespace,
2108 resourceType: generation.resourceType,
2109 instanceId: generation.instanceId,
2110 downstream: generation.downstream,
2111 props: generation.props,
2112 attr: generation.attr,
2113 // A missing provider is fatal — plan already dies on zombie
2114 // rows (see the deletions builder in Plan.ts); this guards
2115 // the replaced-chain generations that bypass plan. The old
2116 // generation is torn down with the provider variant of the
2117 // mode that created it (local ⇄ live replacements);
2118 // unstamped rows are physically live unless their attrs
2119 // carry the `dev:` identity marker (see stampedMode).
2120 provider: yield* tryFindProviderByType(
2121 generation.resourceType,
2122 stampedMode(generation),
2123 ).pipe(
2124 Effect.flatMap(
2125 Option.match({
2126 onNone: () =>
2127 Effect.die(missingProviderError(generation.resourceType, node.fqn)),
2128 onSome: Effect.succeed,
2129 }),
2130 ),
2131 ),
2132 providerMode: generation.providerMode,
2133 };
2134
2135 const adoptionBlocked = isDeleteNode(node)
2136 ? node.state.adoptionBlocked
2137 : generation.adoptionBlocked;
2138 const incompleteProps = isDeleteNode(node)
2139 ? node.state.propsIncomplete
2140 : generation.propsIncomplete;
2141 // Mutable: an attr-less row (interrupted create) may recover its
2142 // attributes from `provider.read` below, right before deletion.
2143 let attr = persistedAttr;
2144
2145 const nextAncestors = new Set(ancestors).add(fqn);
2146
2147 const commit = <S extends ResourceState>(value: Omit<S, "namespace">) =>
2148 state.set({
2149 stack: stackName,
2150 stage,
2151 fqn,
2152 // Same rule as the lifecycle commit above: state only stores
2153 // plain data, never unresolved Output exprs or Effect leaves.
2154 value: {
2155 ...value,
2156 adoptionBlocked:
2157 value.adoptionBlocked ??
2158 (isDeleteNode(node) ? node.state.adoptionBlocked : node.adoptionBlocked),
2159 props: stripUnresolved(value.props),
2160 bindings: stripUnresolved(value.bindings),
2161 namespace,
2162 } as S,
2163 });
2164
2165 const report = (status: ApplyStatus) =>
2166 reportEvent({
2167 type: "apply.resource.status",
2168 fqn,
2169 id: logicalId,
2170 resourceType,
2171 status,
2172 providerMode,
2173 });
2174
2175 const scopedSession = resourceSession(reportEvent, fqn, logicalId);
2176
2177 return yield* (deletions[fqn] ??= yield* Effect.cached(
2178 Effect.gen(function* () {
2179 // Dependents (`downstream`) are deleted before this resource. A
2180 // dependent whose delete failed (or was itself blocked) may make
2181 // this resource legitimately undeletable (dependency violation),
2182 // so it is skipped with a "blocked by" note instead of surfacing
2183 // a spurious second error.
2184 const dependents = yield* Effect.forEach(
2185 downstream,
2186 (dep) =>
2187 dep !== fqn && !ancestors.has(dep)
2188 ? dep in deletionGraph
2189 ? deleteResource(
2190 deletionGraph[dep] as Delete | ReplacementResourceState,
2191 nextAncestors,
2192 ).pipe(Effect.map((outcome) => ({ dep, outcome })))
2193 : // Not in this pass's graph — but it may have failed in
2194 // an earlier drain pass of the same destroy.
2195 Effect.sync(() => ({
2196 dep,
2197 outcome: unresolved.has(dep) ? ("failed" as const) : ("deleted" as const),
2198 }))
2199 : Effect.succeed({ dep, outcome: "deleted" as const }),
2200 { concurrency: "unbounded" },
2201 );
2202
2203 const blockedBy = dependents
2204 .filter(({ outcome }) => outcome === "failed" || outcome === "blocked")
2205 .map(({ dep }) => dep);
2206
2207 if (blockedBy.length > 0) {
2208 unresolved.add(fqn);
2209 blockedDeletes.push({ fqn, logicalId, resourceType, blockedBy });
2210 yield* scopedSession.note(
2211 `Skipping delete — blocked by failed delete of ${blockedBy.join(", ")}.`,
2212 );
2213 yield* report("skipped");
2214 return "blocked" as const;
2215 }
2216
2217 if (
2218 dependents.some(
2219 ({ outcome }) =>
2220 outcome === "deferred" || (isDeleteNode(node) && outcome === "pending"),
2221 )
2222 ) {
2223 // Even GC must wait when the dependent's physical delete has not run.
2224 return "deferred" as const;
2225 }
2226
2227 return yield* deleteResourceBody().pipe(
2228 Effect.catchCause((cause) =>
2229 Effect.gen(function* () {
2230 unresolved.add(fqn);
2231 failures.push({ fqn, logicalId, resourceType, cause });
2232 yield* report("fail");
2233 return "failed" as const;
2234 }),
2235 ),
2236 );
2237 }),
2238 ));
2239
2240 function deleteResourceBody() {
2241 return Effect.gen(function* () {
2242 if (isDeleteNode(node)) {
2243 yield* report(node.action === "orphaned" ? "orphaning" : "deleting");
2244 if (node.action === "orphaned") {
2245 yield* state.delete({ stack: stackName, stage, fqn });
2246 yield* report("orphaned");
2247 delete pendingDeletes[fqn];
2248 // Retention is intentional — it never blocks dependencies.
2249 return "deleted" as const;
2250 }
2251 }
2252
2253 // Honor `retain` for the old generation of a replacement, mirroring
2254 // the orphan-delete path above. Delete-node retain is already
2255 // handled with an early return; this guards the replaced
2256 // old-generation physical delete.
2257 const retainOldGeneration = !isDeleteNode(node) && node.removalPolicy === "retain";
2258
2259 if (retainOldGeneration) {
2260 yield* scopedSession.note("Retaining replaced resource (removal policy: retain)...");
2261 }
2262
2263 // A row can reach deletion with `attr === undefined` when a create
2264 // was interrupted after the cloud-side call succeeded but before
2265 // `reconcile` returned Attributes (a `creating` row — or the old
2266 // generation of a replacement chain in the same predicament).
2267 // Skipping the provider's delete outright would silently orphan
2268 // the physical resource, so ask `read` to look it up from the
2269 // persisted props (providers derive the deterministic physical
2270 // name from id/props):
2271 // - plain attrs → exists and is ours; proceed to delete
2272 // - Unowned(attrs) → exists but is NOT ours (e.g. our create
2273 // actually lost a name race, or died before
2274 // stamping ownership) — never delete a
2275 // foreign resource; drop our state and say so
2276 // - undefined → nothing exists; dropping state is safe
2277 if (attr === undefined && !retainOldGeneration) {
2278 if (adoptionBlocked === "migrated-fqn") {
2279 yield* scopedSession.note(
2280 "Skipping recovery of a reused FQN: a lookup could find its " +
2281 "migrated predecessor. Any interrupted new physical " +
2282 "resource must be identified and cleaned up manually.",
2283 );
2284 } else if (incompleteProps) {
2285 yield* scopedSession.note(
2286 "Skipping recovery: unresolved properties were omitted from " +
2287 "this interrupted create's state, so they cannot identify it. " +
2288 "Any interrupted physical resource must be identified and " +
2289 "cleaned up manually.",
2290 );
2291 } else if (provider.read) {
2292 const recovered = yield* provider
2293 .read({ id: logicalId, fqn, instanceId, olds: props as never, output: undefined })
2294 .pipe(
2295 instrumentLifecycle("read", fqn, resourceType, logicalId, instanceId),
2296 // The persisted props of an interrupted create can carry
2297 // holes where unresolved Outputs were stripped at commit
2298 // time (see stripUnresolved) — e.g. a parent reference
2299 // persisted as `{}`. A provider that dereferences one
2300 // crashes deep inside its SDK client (a SchemaError
2301 // defect), which would make the stage impossible to
2302 // destroy. Recovery is best-effort: degrade the defect
2303 // to "nothing recovered", surface a note, and let the
2304 // row be dropped (#995).
2305 Effect.catchDefect((defect) =>
2306 scopedSession
2307 .note(
2308 "Recovery read crashed while looking up this " +
2309 "resource's interrupted create " +
2310 `(${String(defect)}) — if a physical resource ` +
2311 "was created, it must be cleaned up manually.",
2312 )
2313 .pipe(Effect.as(undefined)),
2314 ),
2315 );
2316 if (recovered !== undefined) {
2317 if (Unowned.is(recovered)) {
2318 yield* scopedSession.note(
2319 "Resource exists in the cloud but is not owned by this " +
2320 "stack — leaving it in place (re-deploy with --adopt " +
2321 "to take ownership, then destroy).",
2322 );
2323 } else {
2324 attr = stripUnowned(recovered as Record<string, any>);
2325 }
2326 }
2327 } else {
2328 yield* scopedSession.note(
2329 "No attributes were recorded for this resource (its create " +
2330 "was interrupted) and the provider does not implement " +
2331 "`read` — if a physical resource was created, it must be " +
2332 "cleaned up manually.",
2333 );
2334 }
2335 }
2336
2337 if (isDeleteNode(node)) {
2338 yield* commit<DeletingResourceState>({
2339 status: "deleting",
2340 fqn,
2341 logicalId,
2342 instanceId,
2343 resourceType,
2344 props,
2345 attr,
2346 downstream,
2347 providerVersion: provider.version ?? 0,
2348 bindings: excludeDeletedBindings(node.bindings),
2349 removalPolicy: node.resource.RemovalPolicy,
2350 providerMode,
2351 adoptionBlocked,
2352 propsIncomplete: incompleteProps,
2353 });
2354 }
2355
2356 if (attr !== undefined && !retainOldGeneration) {
2357 // A destroy must finish now; only a replacement's old generation can wait for the next apply.
2358 const canFinishLater = !isDeleteNode(node) && pendingDeletes[fqn] === undefined;
2359 const deleteInProgress = yield* provider
2360 .delete({
2361 id: logicalId,
2362 fqn,
2363 instanceId,
2364 olds: props as never,
2365 output: attr,
2366 session: scopedSession,
2367 bindings: [],
2368 })
2369 .pipe(
2370 instrumentLifecycle("delete", fqn, resourceType, logicalId, instanceId),
2371 Effect.as(undefined),
2372 Effect.catchIf(
2373 (error): error is DeleteInProgress =>
2374 canFinishLater && Predicate.isTagged(error, "DeleteInProgress"),
2375 (error) => Effect.succeed(error),
2376 ),
2377 );
2378 if (deleteInProgress !== undefined) {
2379 inProgress.add(instanceId);
2380 yield* scopedSession.note(
2381 `${deleteInProgress.message} The replaced resource stays in state and its delete is retried on the next apply.`,
2382 );
2383 return "deferred" as const;
2384 }
2385 }
2386
2387 if (isDeleteNode(node)) {
2388 yield* state.delete({ stack: stackName, stage, fqn });
2389 yield* report("deleted");
2390 } else {
2391 if (!retainOldGeneration) {
2392 yield* scopedSession.note("Cleaning up replaced resource...");
2393 }
2394 if (generation !== node.old) {
2395 yield* commit<ReplacementResourceState>({
2396 ...node,
2397 bindings: excludeDeletedBindings(node.bindings),
2398 old: withoutGeneration(node.old, generation),
2399 });
2400 } else if (node.old.status === "replacing" || node.old.status === "replaced") {
2401 const remaining: ReplacementResourceState = {
2402 ...node,
2403 bindings: excludeDeletedBindings(node.bindings),
2404 old: node.old.old,
2405 };
2406 yield* commit(remaining);
2407 } else {
2408 // Precreate attributes do not mean the replacement reconcile completed.
2409 yield* commit<CreatedResourceState | CreatingResourceState>({
2410 ...(node.status === "replacing"
2411 ? { status: "creating" as const, attr: node.attr }
2412 : { status: "created" as const, attr: node.attr }),
2413 fqn,
2414 logicalId: node.logicalId,
2415 instanceId: node.instanceId,
2416 resourceType: node.resourceType,
2417 props: node.props,
2418 providerVersion: node.providerVersion,
2419 downstream: node.downstream,
2420 bindings: excludeDeletedBindings(node.bindings),
2421 removalPolicy: node.removalPolicy,
2422 providerMode: node.providerMode,
2423 propsIncomplete: node.propsIncomplete,
2424 });
2425 }
2426 yield* scopedSession.note(
2427 retainOldGeneration
2428 ? "Replaced resource retained."
2429 : "Replaced resource cleanup complete.",
2430 );
2431 }
2432 if (isDeleteNode(node)) {
2433 delete pendingDeletes[fqn];
2434 return "deleted" as const;
2435 }
2436 return node.old.status === "replacing" ||
2437 node.old.status === "replaced" ||
2438 pendingDeletes[fqn] !== undefined
2439 ? ("pending" as const)
2440 : ("deleted" as const);
2441 });
2442 }
2443 });
2444
2445 // Attempt every root. Per-node failures were recorded (and their state
2446 // retained) inside each node's delete effect — the effects resolve to
2447 // outcomes and never fail, so one bad resource never interrupts sibling
2448 // deletions mid-flight.
2449 yield* Effect.forEach(
2450 Object.values(deletionGraph).filter((node) => node !== undefined),
2451 (node) => deleteResource(node),
2452 { concurrency: "unbounded" },
2453 );
2454 });
2455
2456 // Drain one old generation per FQN per pass, preserving that generation's
2457 // dependency edges. Planned deletions remain queued until their chains drain.
2458 while (true) {
2459 const remainingReplacedResources = (yield* state.getReplacedResources({
2460 stack: stackName,
2461 stage,
2462 })).filter((replaced) => {
2463 if (unresolved.has(replaced.fqn)) return false;
2464 if (plan.selectedFqns !== undefined && !plan.selectedFqns.has(replaced.fqn)) {
2465 return false;
2466 }
2467 if (nextOldGeneration(replaced, inProgress) === undefined) {
2468 // Only draining generations remain: keep them for the next apply and block dependent deletes.
2469 unresolved.add(replaced.fqn);
2470 return false;
2471 }
2472 return true;
2473 });
2474 const deletionGraph: Record<string, Delete | ReplacementResourceState | undefined> =
2475 Object.fromEntries(remainingReplacedResources.map((replaced) => [replaced.fqn, replaced]));
2476 for (const [fqn, node] of Object.entries(pendingDeletes)) {
2477 if (node === undefined || unresolved.has(fqn)) continue;
2478 const current = yield* state.get({ stack: stackName, stage, fqn });
2479 if (current === undefined) {
2480 delete pendingDeletes[fqn];
2481 } else if (current.kind !== "action") {
2482 deletionGraph[fqn] =
2483 node.action !== "orphaned" &&
2484 (current.status === "replacing" || current.status === "replaced")
2485 ? current
2486 : { ...node, state: current, downstream: current.downstream };
2487 }
2488 }
2489 if (Object.keys(deletionGraph).length === 0) break;
2490 yield* deleteGraph(deletionGraph);
2491 }
2492
2493 if (failures.length > 0) {
2494 // Every independent delete was still attempted; now surface everything
2495 // that went wrong (and everything skipped as a consequence) as one
2496 // typed aggregate. The destroy as a whole still fails.
2497 return yield* new DestroyError({ failures, blocked: blockedDeletes });
2498 }
2499 });
2500
2501 /** The newest old generation of `row` whose instance ID is not in `skip`. */
2502 const nextOldGeneration = (
2503 row: ReplacementResourceState,
2504 skip: ReadonlySet<string>,
2505 ): ReplacementOldResourceState | undefined => {
2506 let generation = row.old;
2507 while (skip.has(generation.instanceId)) {
2508 if (generation.status !== "replacing" && generation.status !== "replaced") {
2509 return undefined;
2510 }
2511 generation = generation.old;
2512 }
2513 return generation;
2514 };
2515
2516 /** The chain below `generation` without `target`; a wrapper left with no old generation becomes a plain row. */
2517 const withoutGeneration = (
2518 generation: ReplacementOldResourceState,
2519 target: ResourceState,
2520 ): ReplacementOldResourceState => {
2521 if (generation.status !== "replacing" && generation.status !== "replaced") {
2522 return generation;
2523 }
2524 if (generation.old !== target) {
2525 return { ...generation, old: withoutGeneration(generation.old, target) };
2526 }
2527 if (target.status === "replacing" || target.status === "replaced") {
2528 return { ...generation, old: target.old };
2529 }
2530 const { old: _old, deleteFirst: _deleteFirst, ...rest } = generation;
2531 return generation.status === "replacing"
2532 ? { ...rest, status: "creating" }
2533 : { ...rest, status: "created", attr: generation.attr };
2534 };
2535
2536 const excludeDeletedBindings = (
2537 bindings: ReadonlyArray<ResourceBinding & { action?: string }>,
2538 ): ResourceBinding[] =>
2539 bindings.flatMap(({ action, sid, data }) => (action === "delete" ? [] : [{ sid, data }]));
2540
2541 /** Order-insensitive equality of two FQN lists. */
2542 const sameSet = (a: ReadonlyArray<string>, b: ReadonlyArray<string>) => {
2543 const sa = new Set(a);
2544 return sa.size === new Set(b).size && b.every((fqn) => sa.has(fqn));
2545 };