The best way to control Claude Code, Codex and more from any computer or phone

main
ts 848 lines 34.9 KB
Raw
1 import * as batch from "@distilled.cloud/aws/batch";
2 import * as ec2 from "@distilled.cloud/aws/ec2";
3 import * as iam from "@distilled.cloud/aws/iam";
4 import { isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { createInternalTags, diffTags } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import { AWSEnvironment, type AccountID } from "../Environment.ts";
14 import type { Providers } from "../Providers.ts";
15 import type { RegionID } from "../Region.ts";
16 import { pollBatch, retryBatch } from "./internal.ts";
17
18 export type ComputeEnvironmentName = string;
19 export type ComputeEnvironmentArn =
20 `arn:aws:batch:${RegionID}:${AccountID}:compute-environment/${ComputeEnvironmentName}`;
21
22 /**
23 * Raised when networking is left implicit but the account/region has no
24 * default VPC to fall back to.
25 */
26 export class NoDefaultVpcError extends Data.TaggedError("NoDefaultVpcError")<{
27 readonly message: string;
28 }> {}
29
30 /**
31 * Raised when AWS Batch finishes reconciling a compute environment in an
32 * unusable state. The status reason is preserved so callers see the actual
33 * AWS configuration or dependency failure instead of receiving stale
34 * `status: "INVALID"` attributes from a successful deployment.
35 */
36 export class ComputeEnvironmentInvalidError extends Data.TaggedError(
37 "ComputeEnvironmentInvalidError",
38 )<{
39 readonly computeEnvironmentName: string;
40 readonly status: string;
41 readonly statusReason: string | undefined;
42 readonly message: string;
43 }> {}
44
45 export interface ComputeEnvironmentProps {
46 /**
47 * Name of the compute environment. If omitted, a unique name is generated.
48 * Up to 128 characters (letters, numbers, hyphens, underscores).
49 */
50 computeEnvironmentName?: string;
51 /**
52 * Whether AWS Batch manages the compute capacity. Unmanaged environments
53 * are useful when capacity is registered separately, and do not require
54 * subnets or security groups.
55 * Changing this replaces the compute environment.
56 * @default "MANAGED"
57 */
58 managementType?: "MANAGED" | "UNMANAGED";
59 /**
60 * Fargate capacity type for the managed compute environment.
61 * Changing this replaces the compute environment.
62 * @default "FARGATE"
63 */
64 type?: "FARGATE" | "FARGATE_SPOT";
65 /**
66 * Maximum number of Fargate vCPUs the environment can scale to.
67 * @default 4
68 */
69 maxvCpus?: number;
70 /**
71 * Number of externally-managed vCPUs available to an unmanaged compute
72 * environment.
73 * @default 4
74 */
75 unmanagedvCpus?: number;
76 /**
77 * VPC subnets the Fargate tasks run in. If omitted, the default VPC's
78 * subnets are used.
79 */
80 subnets?: string[];
81 /**
82 * Security groups for the Fargate tasks. If omitted, the default VPC's
83 * default security group is used.
84 */
85 securityGroupIds?: string[];
86 /**
87 * Whether the compute environment accepts jobs from associated queues.
88 * @default "ENABLED"
89 */
90 state?: "ENABLED" | "DISABLED";
91 /**
92 * Service role ARN. If omitted, AWS Batch uses (and auto-creates on first
93 * use) the `AWSServiceRoleForBatch` service-linked role.
94 */
95 serviceRole?: string;
96 /**
97 * User-defined tags to apply to the compute environment.
98 */
99 tags?: Record<string, string>;
100 }
101
102 export interface ComputeEnvironment extends Resource<
103 "AWS.Batch.ComputeEnvironment",
104 ComputeEnvironmentProps,
105 {
106 computeEnvironmentName: ComputeEnvironmentName;
107 computeEnvironmentArn: ComputeEnvironmentArn;
108 ecsClusterArn: string | undefined;
109 managementType: "MANAGED" | "UNMANAGED";
110 type: "FARGATE" | "FARGATE_SPOT";
111 state: "ENABLED" | "DISABLED";
112 status: string;
113 maxvCpus: number;
114 unmanagedvCpus: number;
115 subnets: string[];
116 securityGroupIds: string[];
117 tags: Record<string, string>;
118 },
119 never,
120 Providers
121 > {}
122
123 /**
124 * An AWS Batch managed compute environment backed by Fargate (or Fargate
125 * Spot) capacity. Fargate compute environments provision in seconds and
126 * require no instance management.
127 *
128 * ### Creating Compute Environments
129 * **Example:** Default Fargate Compute Environment
130 * ```typescript
131 * // Uses the default VPC's subnets and default security group.
132 * const ce = yield* Batch.ComputeEnvironment("JobsCE", {});
133 * ```
134 *
135 * **Example:** Unmanaged Compute Environment
136 * ```typescript
137 * const ce = yield* Batch.ComputeEnvironment("ExternalCapacity", {
138 * managementType: "UNMANAGED",
139 * unmanagedvCpus: 8,
140 * });
141 * ```
142 *
143 * **Example:** Fargate Spot with explicit networking
144 * ```typescript
145 * const ce = yield* Batch.ComputeEnvironment("SpotCE", {
146 * type: "FARGATE_SPOT",
147 * maxvCpus: 16,
148 * subnets: [subnetA.subnetId, subnetB.subnetId],
149 * securityGroupIds: [sg.groupId],
150 * });
151 * ```
152 *
153 * ### Composing the Batch chain
154 * **Example:** Compute Environment → Job Queue
155 * ```typescript
156 * const ce = yield* Batch.ComputeEnvironment("JobsCE", {});
157 * const queue = yield* Batch.JobQueue("JobsQueue", {
158 * computeEnvironments: [ce.computeEnvironmentArn],
159 * });
160 * ```
161 *
162 * @resource
163 */
164 export const ComputeEnvironment = Resource<ComputeEnvironment>("AWS.Batch.ComputeEnvironment");
165
166 const toAttributes = (ce: batch.ComputeEnvironmentDetail, tags: Record<string, string>) => ({
167 computeEnvironmentName: ce.computeEnvironmentName!,
168 computeEnvironmentArn: ce.computeEnvironmentArn as ComputeEnvironmentArn,
169 ecsClusterArn: ce.ecsClusterArn,
170 managementType: (ce.type ?? "MANAGED") as "MANAGED" | "UNMANAGED",
171 type: (ce.computeResources?.type ?? "FARGATE") as "FARGATE" | "FARGATE_SPOT",
172 state: (ce.state ?? "ENABLED") as "ENABLED" | "DISABLED",
173 status: ce.status ?? "VALID",
174 maxvCpus: ce.computeResources?.maxvCpus ?? 0,
175 unmanagedvCpus: ce.unmanagedvCpus ?? 0,
176 subnets: [...(ce.computeResources?.subnets ?? [])],
177 securityGroupIds: [...(ce.computeResources?.securityGroupIds ?? [])],
178 tags,
179 });
180
181 const observedTagsOf = (ce: { tags?: { [key: string]: string | undefined } }) =>
182 Object.fromEntries(
183 Object.entries(ce.tags ?? {}).filter((e): e is [string, string] => typeof e[1] === "string"),
184 );
185
186 /**
187 * IAM eventual-consistency race: a CE validated against a service role (or
188 * its policy attachment) created moments earlier in the same deploy can
189 * settle INVALID with a CLIENT_ERROR authorization statusReason (e.g.
190 * "... is not authorized to perform: ecs:DescribeClusters ...
191 * AccessDeniedException") even though the role and policy are correct.
192 * Batch only re-validates an INVALID CE on its slow periodic health check
193 * (tens of minutes), so waiting on the same CE cannot recover in deploy
194 * time — the recovery is to delete the freshly-created CE and re-create.
195 */
196 const isAuthPropagationInvalid = (ce: batch.ComputeEnvironmentDetail | undefined): boolean =>
197 ce?.status === "INVALID" &&
198 ce.statusReason !== undefined &&
199 ce.statusReason.includes("CLIENT_ERROR") &&
200 (ce.statusReason.includes("is not authorized to perform") ||
201 ce.statusReason.includes("AccessDeniedException") ||
202 ce.statusReason.includes("AccessDenied"));
203
204 /**
205 * Resolve the default VPC's subnets and default security group — used when
206 * the user doesn't pin networking explicitly.
207 */
208 const resolveDefaultNetwork = Effect.gen(function* () {
209 const vpcs = yield* ec2.describeVpcs({
210 Filters: [{ Name: "is-default", Values: ["true"] }],
211 });
212 const vpcId = vpcs.Vpcs?.find(
213 (vpc) => vpc.State === undefined || vpc.State === "available",
214 )?.VpcId;
215 if (!vpcId) {
216 return yield* new NoDefaultVpcError({
217 message: "No default VPC found — pass `subnets` and `securityGroupIds` explicitly",
218 });
219 }
220 const subnets = yield* ec2.describeSubnets
221 .items({
222 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
223 })
224 .pipe(
225 Stream.runCollect,
226 Effect.map((chunk) => Array.from(chunk)),
227 );
228 const groups = yield* ec2.describeSecurityGroups({
229 Filters: [
230 { Name: "vpc-id", Values: [vpcId] },
231 { Name: "group-name", Values: ["default"] },
232 ],
233 });
234 const network = {
235 subnets: subnets.flatMap((s) =>
236 s.SubnetId && (s.State === undefined || s.State === "available") ? [s.SubnetId] : [],
237 ),
238 securityGroupIds: (groups.SecurityGroups ?? []).flatMap((g) => (g.GroupId ? [g.GroupId] : [])),
239 };
240 if (network.subnets.length === 0 || network.securityGroupIds.length === 0) {
241 return yield* new NoDefaultVpcError({
242 message:
243 "Default VPC networking is not ready — pass `subnets` and `securityGroupIds` explicitly",
244 });
245 }
246 return network;
247 }).pipe(
248 Effect.retry({
249 while: (error) => error._tag === "NoDefaultVpcError",
250 schedule: Schedule.max([Schedule.spaced("3 seconds"), Schedule.recurs(10)]),
251 }),
252 );
253
254 export const ComputeEnvironmentProvider = () =>
255 Provider.effect(
256 ComputeEnvironment,
257 Effect.gen(function* () {
258 const toName = (id: string, props: ComputeEnvironmentProps = {}) =>
259 props.computeEnvironmentName
260 ? Effect.succeed(props.computeEnvironmentName)
261 : createPhysicalName({ id, maxLength: 128 });
262
263 // A deleted environment lingers as a DELETED record for a while and the
264 // name is immediately reusable, so a describe-by-name can briefly
265 // return both the tombstone and the live one — prefer the live one.
266 const describeOne = (name: string) =>
267 batch
268 .describeComputeEnvironments({ computeEnvironments: [name] })
269 .pipe(
270 Effect.map(
271 (res) =>
272 res.computeEnvironments?.find((ce) => ce.status !== "DELETED") ??
273 res.computeEnvironments?.[0],
274 ),
275 );
276
277 /** Wait until any CREATING/UPDATING/DELETING transition settles. */
278 const awaitSettled = (name: string) =>
279 pollBatch(
280 describeOne(name),
281 (ce) =>
282 ce === undefined ||
283 (ce.status !== "CREATING" && ce.status !== "UPDATING" && ce.status !== "DELETING"),
284 );
285
286 const awaitDisabled = (name: string) =>
287 pollBatch(
288 describeOne(name),
289 (ce) =>
290 ce === undefined ||
291 ce.status === "DELETED" ||
292 ce.status === "DELETING" ||
293 (ce.state === "DISABLED" && ce.status !== "CREATING" && ce.status !== "UPDATING"),
294 );
295
296 const invalid = (name: string, ce: batch.ComputeEnvironmentDetail | undefined) => {
297 const status = ce?.status ?? "MISSING";
298 const statusReason = ce?.statusReason;
299 return new ComputeEnvironmentInvalidError({
300 computeEnvironmentName: name,
301 status,
302 statusReason,
303 message: `Compute environment ${name} settled as ${status}${statusReason ? `: ${statusReason}` : ""}`,
304 });
305 };
306
307 /**
308 * Restore the CE's regular (non-service-linked) service role when it no
309 * longer exists in IAM, returning the role name to drop afterwards.
310 * Batch cannot tear a CE down without assuming this role — a dangling
311 * role is the canonical cause of undeletable, forever-INVALID CEs.
312 */
313 const restoreServiceRoleIfMissing = Effect.fn(function* (serviceRoleArn: string | undefined) {
314 if (!serviceRoleArn || serviceRoleArn.includes("/aws-service-role/")) {
315 return undefined;
316 }
317 const roleName = serviceRoleArn.split("/").pop()!;
318 const exists = yield* iam.getRole({ RoleName: roleName }).pipe(
319 Effect.map(() => true),
320 Effect.catchTag("NoSuchEntityException", () => Effect.succeed(false)),
321 );
322 if (exists) return undefined;
323 yield* iam
324 .createRole({
325 RoleName: roleName,
326 AssumeRolePolicyDocument: JSON.stringify({
327 Version: "2012-10-17",
328 Statement: [
329 {
330 Effect: "Allow",
331 Principal: { Service: "batch.amazonaws.com" },
332 Action: "sts:AssumeRole",
333 },
334 ],
335 }),
336 Description:
337 "Temporarily restored by alchemy so AWS Batch can tear down a compute environment whose service role was deleted",
338 })
339 .pipe(
340 // Concurrent delete of a replaced instance may restore it first.
341 Effect.catchTag("EntityAlreadyExistsException", () => Effect.void),
342 );
343 yield* iam.attachRolePolicy({
344 RoleName: roleName,
345 PolicyArn: "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole",
346 });
347 // IAM propagation before Batch tries to assume the role.
348 yield* Effect.sleep("10 seconds");
349 return roleName;
350 });
351
352 /** Drop a role restored by {@link restoreServiceRoleIfMissing}. */
353 const dropRestoredServiceRole = Effect.fn(function* (roleName: string) {
354 yield* iam
355 .detachRolePolicy({
356 RoleName: roleName,
357 PolicyArn: "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole",
358 })
359 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
360 yield* iam
361 .deleteRole({ RoleName: roleName })
362 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
363 });
364
365 /**
366 * A force nuke cannot rely on stack dependency ordering: a listed
367 * compute environment may still be attached to a separately-listed job
368 * queue. Delete those blockers first. Ordinary stack deletion never
369 * touches out-of-band queues.
370 */
371 const deleteRelatedJobQueues = (computeEnvironment: string) =>
372 Effect.gen(function* () {
373 const pages = yield* batch.describeJobQueues.pages({}).pipe(Stream.runCollect);
374 const queues = Array.from(pages)
375 .flatMap((page) => page.jobQueues ?? [])
376 .filter(
377 (queue) =>
378 queue.jobQueueName &&
379 queue.status !== "DELETED" &&
380 queue.computeEnvironmentOrder?.some(
381 (entry) =>
382 entry.computeEnvironment === computeEnvironment ||
383 entry.computeEnvironment?.endsWith(
384 `:compute-environment/${computeEnvironment}`,
385 ),
386 ),
387 );
388
389 yield* Effect.forEach(
390 queues,
391 (queue) =>
392 Effect.gen(function* () {
393 const queueName = queue.jobQueueName!;
394 const describeQueue = batch
395 .describeJobQueues({ jobQueues: [queueName] })
396 .pipe(
397 Effect.map((response) =>
398 response.jobQueues?.find((candidate) => candidate.status !== "DELETED"),
399 ),
400 );
401 let observed = yield* describeQueue;
402 if (!observed) return;
403 if (
404 observed.status !== "DELETING" &&
405 (observed.state ?? "ENABLED") !== "DISABLED"
406 ) {
407 yield* retryBatch(
408 batch.updateJobQueue({
409 jobQueue: queueName,
410 state: "DISABLED",
411 }),
412 (error) => error._tag === "JobQueueBeingModified",
413 ).pipe(Effect.catchTag("JobQueueNotFound", () => Effect.void));
414 }
415 observed = yield* pollBatch(
416 describeQueue,
417 (candidate) =>
418 candidate === undefined ||
419 candidate.status === "DELETED" ||
420 candidate.status === "DELETING" ||
421 (candidate.state === "DISABLED" &&
422 (candidate.status === "VALID" || candidate.status === "INVALID")),
423 );
424 if (!observed || observed.status === "DELETED") return;
425 if (observed.status !== "DELETING") {
426 yield* retryBatch(
427 batch.deleteJobQueue({ jobQueue: queueName }),
428 (error) => error._tag === "JobQueueBeingModified",
429 ).pipe(Effect.catchTag("JobQueueNotFound", () => Effect.void));
430 }
431 yield* pollBatch(
432 describeQueue,
433 (candidate) => candidate === undefined || candidate.status === "DELETED",
434 );
435 }),
436 { concurrency: 4, discard: true },
437 );
438 });
439
440 return {
441 stables: ["computeEnvironmentName", "computeEnvironmentArn"],
442 diff: Effect.fn(function* ({ id, olds, news }) {
443 if (!isResolved(news)) return;
444 if ((yield* toName(id, olds ?? {})) !== (yield* toName(id, news ?? {}))) {
445 return { action: "replace" } as const;
446 }
447 if ((olds?.type ?? "FARGATE") !== (news?.type ?? "FARGATE")) {
448 return { action: "replace" } as const;
449 }
450 if ((olds?.managementType ?? "MANAGED") !== (news?.managementType ?? "MANAGED")) {
451 return { action: "replace" } as const;
452 }
453 }),
454 read: Effect.fn(function* ({ id, olds, output }) {
455 const name = output?.computeEnvironmentName ?? (yield* toName(id, olds ?? {}));
456 const ce = yield* describeOne(name);
457 if (!ce?.computeEnvironmentArn || ce.status === "DELETED") {
458 return undefined;
459 }
460 return toAttributes(ce, observedTagsOf(ce));
461 }),
462 list: () =>
463 Effect.gen(function* () {
464 const pages = yield* batch.describeComputeEnvironments
465 .pages({})
466 .pipe(Stream.runCollect);
467 return Array.from(pages)
468 .flatMap((page) => page.computeEnvironments ?? [])
469 .flatMap((ce) =>
470 ce.computeEnvironmentArn && ce.status !== "DELETED"
471 ? [toAttributes(ce, observedTagsOf(ce))]
472 : [],
473 );
474 }),
475 reconcile: Effect.fn(function* ({ id, news = {}, output, session }) {
476 const { accountId, region } = yield* AWSEnvironment.current;
477 const name = output?.computeEnvironmentName ?? (yield* toName(id, news));
478 const arn =
479 `arn:aws:batch:${region}:${accountId}:compute-environment/${name}` as ComputeEnvironmentArn;
480 const internalTags = yield* createInternalTags(id);
481 const desiredTags = { ...internalTags, ...news.tags };
482 const desiredState = news.state ?? "ENABLED";
483 const desiredManagementType = news.managementType ?? "MANAGED";
484 const desiredMaxvCpus = news.maxvCpus ?? 4;
485 const desiredUnmanagedvCpus = news.unmanagedvCpus ?? 4;
486 // Implicit networking means the *current* default VPC network, not
487 // whichever subnet IDs happened to exist at create time. Account
488 // cleanup can recreate the default VPC between reconciliations, so
489 // resolve the desired network on every pass and repair stale IDs.
490 const fallbackNetwork =
491 desiredManagementType === "MANAGED" && (!news.subnets || !news.securityGroupIds)
492 ? yield* resolveDefaultNetwork
493 : undefined;
494 let desiredNetwork =
495 desiredManagementType === "MANAGED"
496 ? {
497 subnets: news.subnets ?? fallbackNetwork!.subnets,
498 securityGroupIds: news.securityGroupIds ?? fallbackNetwork!.securityGroupIds,
499 }
500 : undefined;
501 const sameSet = (a: readonly string[], b: readonly string[]) =>
502 a.length === b.length && [...a].sort().join(",") === [...b].sort().join(",");
503 const matchesDesired = (candidate: batch.ComputeEnvironmentDetail) =>
504 (candidate.type ?? "MANAGED") === desiredManagementType &&
505 (candidate.state ?? "ENABLED") === desiredState &&
506 (desiredManagementType === "UNMANAGED"
507 ? (candidate.unmanagedvCpus ?? 0) === desiredUnmanagedvCpus
508 : (candidate.computeResources?.maxvCpus ?? 0) === desiredMaxvCpus &&
509 sameSet(candidate.computeResources?.subnets ?? [], desiredNetwork!.subnets) &&
510 sameSet(
511 candidate.computeResources?.securityGroupIds ?? [],
512 desiredNetwork!.securityGroupIds,
513 ));
514
515 /**
516 * INVALID is terminal until the configuration is changed. It is
517 * commonly caused by default-VPC subnet/security-group churn during
518 * highly concurrent account tests, so waiting on the same observed
519 * configuration cannot recover. Re-resolve implicit networking and
520 * submit a bounded repair update instead.
521 */
522 const awaitValid = Effect.fn(function* () {
523 let lastRepairNetwork: string | undefined;
524 for (let attempt = 0; attempt < 8; attempt++) {
525 const settled = yield* awaitSettled(name);
526 if (settled?.status === "VALID" && matchesDesired(settled)) {
527 return settled;
528 }
529 if (!settled || settled.status === "DELETED" || attempt === 7) {
530 return yield* invalid(name, settled);
531 }
532
533 if (
534 desiredManagementType === "MANAGED" &&
535 (!news.subnets || !news.securityGroupIds)
536 ) {
537 const refreshed = yield* resolveDefaultNetwork;
538 desiredNetwork = {
539 subnets: news.subnets ?? refreshed.subnets,
540 securityGroupIds: news.securityGroupIds ?? refreshed.securityGroupIds,
541 };
542 }
543 const repairNetwork = [
544 desiredManagementType,
545 "|",
546 ...(desiredNetwork
547 ? [
548 ...[...desiredNetwork.subnets].sort(),
549 "|",
550 ...[...desiredNetwork.securityGroupIds].sort(),
551 ]
552 : [desiredUnmanagedvCpus]),
553 ].join(",");
554 if (lastRepairNetwork === repairNetwork) {
555 // EC2 may briefly keep returning a just-deleted default VPC
556 // network. Do not spend the repair budget re-submitting the
557 // same known-invalid IDs; wait for EC2's replacement view.
558 yield* Effect.sleep("5 seconds");
559 continue;
560 }
561 yield* retryBatch(
562 batch.updateComputeEnvironment({
563 computeEnvironment: name,
564 state: desiredState,
565 unmanagedvCpus:
566 desiredManagementType === "UNMANAGED" ? desiredUnmanagedvCpus : undefined,
567 computeResources:
568 desiredManagementType === "MANAGED"
569 ? {
570 maxvCpus: desiredMaxvCpus,
571 subnets: desiredNetwork!.subnets,
572 securityGroupIds: desiredNetwork!.securityGroupIds,
573 }
574 : undefined,
575 }),
576 (error) => error._tag === "ComputeEnvironmentBeingModified",
577 );
578 lastRepairNetwork = repairNetwork;
579 // Describe can briefly return the pre-update record before the
580 // transition becomes visible.
581 yield* Effect.sleep("3 seconds");
582 }
583 return yield* invalid(name, undefined);
584 });
585
586 // Observe — cloud state is authoritative.
587 let ce = yield* describeOne(name);
588 if (ce?.status === "DELETED") ce = undefined;
589
590 // Ensure — create if missing, then wait until the environment
591 // settles to VALID (Fargate CEs settle in seconds).
592 const create = Effect.asVoid(
593 batch.createComputeEnvironment({
594 computeEnvironmentName: name,
595 type: desiredManagementType,
596 state: desiredState,
597 unmanagedvCpus:
598 desiredManagementType === "UNMANAGED" ? desiredUnmanagedvCpus : undefined,
599 computeResources:
600 desiredManagementType === "MANAGED"
601 ? {
602 type: news.type ?? "FARGATE",
603 maxvCpus: desiredMaxvCpus,
604 subnets: desiredNetwork!.subnets,
605 securityGroupIds: desiredNetwork!.securityGroupIds,
606 }
607 : undefined,
608 serviceRole: news.serviceRole,
609 tags: desiredTags,
610 }),
611 );
612
613 /**
614 * Tear down a CE we created in *this* reconcile that settled
615 * INVALID from an IAM propagation race (see
616 * {@link isAuthPropagationInvalid}). The CE carries no user state
617 * yet, so delete-and-recreate is safe. Mirrors the delete
618 * lifecycle's recovery: the async teardown itself can fail once on
619 * the same not-yet-propagated role, in which case the CE re-settles
620 * INVALID and the delete is re-issued.
621 */
622 const reapInvalidCreate = Effect.fn(function* () {
623 yield* retryBatch(
624 batch.updateComputeEnvironment({
625 computeEnvironment: name,
626 state: "DISABLED",
627 }),
628 (e) => e._tag === "ComputeEnvironmentBeingModified",
629 ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
630 const disabled = yield* awaitDisabled(name);
631 if (!disabled || disabled.status === "DELETED") return;
632 const requestDelete = retryBatch(
633 batch.deleteComputeEnvironment({ computeEnvironment: name }),
634 (e) =>
635 e._tag === "ComputeEnvironmentInUse" ||
636 e._tag === "ComputeEnvironmentBeingModified",
637 ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
638 if (disabled.status !== "DELETING") {
639 yield* requestDelete;
640 // Describe can briefly return the pre-delete record before the
641 // DELETING transition becomes visible.
642 yield* Effect.sleep("3 seconds");
643 }
644 const awaitGone = pollBatch(
645 describeOne(name),
646 (c) => c === undefined || c.status === "DELETED" || c.status === "INVALID",
647 );
648 let final = yield* awaitGone;
649 if (final && final.status === "INVALID") {
650 // The async deletion attempt failed (likely the same IAM
651 // propagation lag) — re-issue once and wait again.
652 yield* requestDelete;
653 yield* Effect.sleep("3 seconds");
654 final = yield* awaitGone;
655 }
656 if (final && final.status !== "DELETED") {
657 return yield* invalid(name, final);
658 }
659 });
660
661 /**
662 * Create and wait for the CE to settle, recovering (bounded) from
663 * the freshly-created-INVALID IAM propagation race by deleting the
664 * INVALID CE and re-creating. Each delete cycle takes ~1-2 minutes,
665 * which is itself ample IAM propagation time. A CE that settles
666 * INVALID for any other reason — or is still auth-INVALID after 3
667 * total create attempts — is returned as-is so the caller's
668 * awaitValid path fails loudly with the typed
669 * ComputeEnvironmentInvalidError.
670 */
671 const createAndSettle = Effect.gen(function* () {
672 yield* create;
673 let settled = yield* awaitSettled(name);
674 for (let attempt = 0; attempt < 2 && isAuthPropagationInvalid(settled); attempt++) {
675 yield* reapInvalidCreate();
676 // Small grace beyond the deletion window before re-validating.
677 yield* Effect.sleep("5 seconds");
678 yield* create;
679 settled = yield* awaitSettled(name);
680 }
681 return settled;
682 });
683
684 if (ce?.computeEnvironmentArn && ce.status === "DELETING") {
685 // An interrupted destroy left the environment mid-deletion — let
686 // it finish, then fall through to a fresh create below.
687 ce = yield* awaitSettled(name);
688 if (ce?.status === "DELETED") ce = undefined;
689 }
690 if (!ce?.computeEnvironmentArn) {
691 ce = yield* createAndSettle;
692 } else if (ce.status !== "VALID" && ce.status !== "INVALID") {
693 // A prior modification is still settling — wait before diffing.
694 ce = yield* awaitSettled(name);
695 }
696 if (ce?.status === "DELETED") {
697 // The settle above can still land on a tombstone (deletion that
698 // completed between observe and settle) — one more create pass.
699 ce = yield* createAndSettle;
700 }
701
702 if (!ce?.computeEnvironmentArn) {
703 return yield* Effect.die(new Error(`ComputeEnvironment ${name} did not settle`));
704 }
705
706 // Sync — diff observed against desired, apply only the delta.
707 const update: batch.UpdateComputeEnvironmentRequest = {
708 computeEnvironment: name,
709 };
710 let dirty = false;
711 if ((ce.state ?? "ENABLED") !== desiredState) {
712 update.state = desiredState;
713 dirty = true;
714 }
715 if (desiredManagementType === "UNMANAGED") {
716 if ((ce.unmanagedvCpus ?? 0) !== desiredUnmanagedvCpus) {
717 update.unmanagedvCpus = desiredUnmanagedvCpus;
718 dirty = true;
719 }
720 } else {
721 const resources: batch.ComputeResourceUpdate = {};
722 if ((ce.computeResources?.maxvCpus ?? 0) !== desiredMaxvCpus) {
723 resources.maxvCpus = desiredMaxvCpus;
724 }
725 if (!sameSet(ce.computeResources?.subnets ?? [], desiredNetwork!.subnets)) {
726 resources.subnets = desiredNetwork!.subnets;
727 }
728 if (
729 !sameSet(
730 ce.computeResources?.securityGroupIds ?? [],
731 desiredNetwork!.securityGroupIds,
732 )
733 ) {
734 resources.securityGroupIds = desiredNetwork!.securityGroupIds;
735 }
736 if (Object.keys(resources).length > 0) {
737 update.computeResources = resources;
738 dirty = true;
739 }
740 }
741 if (dirty) {
742 yield* retryBatch(
743 batch.updateComputeEnvironment(update),
744 (e) => e._tag === "ComputeEnvironmentBeingModified",
745 );
746 ce = yield* awaitValid();
747 } else if (ce.status !== "VALID") {
748 ce = yield* awaitValid();
749 }
750
751 // Sync tags — diff against OBSERVED cloud tags.
752 const { upsert, removed } = diffTags(observedTagsOf(ce), desiredTags);
753 if (upsert.length > 0) {
754 yield* batch.tagResource({
755 resourceArn: arn,
756 tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
757 });
758 }
759 if (removed.length > 0) {
760 yield* batch.untagResource({ resourceArn: arn, tagKeys: removed });
761 }
762
763 yield* session.note(arn);
764 return toAttributes(
765 { ...ce, computeEnvironmentName: name, computeEnvironmentArn: arn },
766 desiredTags,
767 );
768 }),
769 delete: Effect.fn(function* ({ output, force }) {
770 const name = output.computeEnvironmentName;
771 const existing = yield* describeOne(name);
772 if (!existing || existing.status === "DELETED") return;
773
774 if (force === true) {
775 yield* deleteRelatedJobQueues(name);
776 }
777
778 // Must be DISABLED before deletion.
779 if (existing.status !== "DELETING" && (existing.state ?? "ENABLED") !== "DISABLED") {
780 yield* retryBatch(
781 batch.updateComputeEnvironment({
782 computeEnvironment: name,
783 state: "DISABLED",
784 }),
785 (e) => e._tag === "ComputeEnvironmentBeingModified",
786 ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
787 }
788 const disabled = yield* awaitDisabled(name);
789 if (!disabled || disabled.status === "DELETED") return;
790
791 // Batch tears the CE down (its ECS cluster, ENIs, …) by assuming the
792 // CE's service role. If that role was deleted first (crashed destroy,
793 // force-nuke ordering), the async deletion fails with an sts:AssumeRole
794 // CLIENT_ERROR and the CE is stuck INVALID forever. Restore a minimal
795 // same-name role (same ARN — role ARNs carry no unique id) so teardown
796 // can proceed, and drop it again once the CE is gone. Service-linked
797 // roles are never restored (they're account infrastructure).
798 const restoredRole = yield* restoreServiceRoleIfMissing(disabled.serviceRole);
799
800 // deleteComputeEnvironment is idempotent (succeeds when missing) but
801 // rejects while a JobQueue association is still tearing down — retry
802 // through that window (queue deletion is asynchronous).
803 const requestDelete = retryBatch(
804 batch.deleteComputeEnvironment({ computeEnvironment: name }),
805 (e) =>
806 e._tag === "ComputeEnvironmentInUse" || e._tag === "ComputeEnvironmentBeingModified",
807 24,
808 ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
809 if (disabled.status !== "DELETING") {
810 yield* requestDelete;
811 // Describe can briefly return the pre-delete record (e.g. a stale
812 // INVALID status) before the DELETING transition becomes visible.
813 yield* Effect.sleep("3 seconds");
814 }
815
816 // Wait until it's actually gone (CE deletion takes ~1-2 minutes).
817 // Settling on INVALID means the async deletion attempt FAILED (e.g.
818 // the service role was dangling until the restore above) — it never
819 // self-recovers, so re-issue the delete once and wait again.
820 const awaitGone = pollBatch(
821 describeOne(name),
822 (ce) => ce === undefined || ce.status === "DELETED" || ce.status === "INVALID",
823 );
824 let final = yield* awaitGone;
825 if (final && final.status === "INVALID") {
826 yield* requestDelete;
827 yield* Effect.sleep("3 seconds");
828 final = yield* awaitGone;
829 } else if (final && final.status !== "DELETED") {
830 // Still DELETING when the poll budget expired — a slow but healthy
831 // teardown gets one more poll round before failing loudly.
832 final = yield* awaitGone;
833 }
834
835 if (restoredRole !== undefined && (final === undefined || final.status === "DELETED")) {
836 yield* dropRestoredServiceRole(restoredRole);
837 }
838
839 // A poll that merely expires MUST NOT report success — the engine
840 // would then delete the CE's dependencies (service role, subnets)
841 // while teardown is still consuming them, wedging the CE for good.
842 if (final && final.status !== "DELETED") {
843 return yield* invalid(name, final);
844 }
845 }),
846 };
847 }),
848 );