@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 727 lines 27.7 KB
Raw
1 import * as emr from "@distilled.cloud/aws/emr";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
8 import { toWireSeconds } from "alchemy/Util/Duration";
9 import * as Data from "effect/Data";
10 import type * as Duration from "effect/Duration";
11 import * as Effect from "effect/Effect";
12 import * as Option from "effect/Option";
13 import * as Schedule from "effect/Schedule";
14 import * as Stream from "effect/Stream";
15 import type { Providers } from "../Providers.ts";
16
17 export class EMRClusterOperationFailed extends Data.TaggedError("EMRClusterOperationFailed")<{
18 readonly message: string;
19 readonly cause?: unknown;
20 }> {}
21
22 export class EMRClusterNotReady extends Data.TaggedError("EMRClusterNotReady")<{
23 readonly message: string;
24 readonly cause?: unknown;
25 }> {}
26
27 export class EMRClusterUnexpectedResponse extends Data.TaggedError("EMRClusterUnexpectedResponse")<{
28 readonly message: string;
29 readonly cause?: unknown;
30 }> {}
31
32 /**
33 * EC2 instance topology for an EMR {@link Cluster} — a master (primary)
34 * instance group plus an optional core instance group, using the uniform
35 * instance-groups configuration.
36 */
37 export interface ClusterInstancesConfig {
38 /**
39 * EC2 instance type of the single master (primary) node. Changing this
40 * replaces the cluster.
41 * @default "m5.xlarge"
42 */
43 masterInstanceType?: string;
44 /**
45 * EC2 instance type of the core nodes. Changing this replaces the cluster.
46 * @default "m5.xlarge"
47 */
48 coreInstanceType?: string;
49 /**
50 * Number of core nodes. `0` launches a master-only cluster. Resizing
51 * between non-zero counts is applied in place; crossing the `0` boundary
52 * (adding or removing the core group entirely) replaces the cluster.
53 * @default 1
54 */
55 coreInstanceCount?: number;
56 /**
57 * VPC subnet the cluster instances launch in. Changing the subnet replaces
58 * the cluster.
59 * @default an EMR-chosen subnet of the account's default VPC
60 */
61 ec2SubnetId?: string;
62 /**
63 * EC2 key pair for SSH access to the master node. Changing this replaces
64 * the cluster.
65 */
66 ec2KeyName?: string;
67 /**
68 * Whether the cluster stays alive (transitions to `WAITING`) after all
69 * steps complete rather than auto-terminating. Updateable in place.
70 * @default true
71 */
72 keepJobFlowAliveWhenNoSteps?: boolean;
73 /**
74 * Whether the cluster's EC2 instances are locked against termination by
75 * API calls or user intervention. Updateable in place (and always lifted
76 * before the provider terminates the cluster on destroy).
77 * @default false
78 */
79 terminationProtected?: boolean;
80 /**
81 * Whether EMR automatically replaces unhealthy core/task nodes.
82 * Updateable in place.
83 */
84 unhealthyNodeReplacement?: boolean;
85 /**
86 * EMR-managed security group for the master node. Changing this replaces
87 * the cluster.
88 * @default a security group created by EMR
89 */
90 emrManagedMasterSecurityGroup?: string;
91 /**
92 * EMR-managed security group for core/task nodes. Changing this replaces
93 * the cluster.
94 * @default a security group created by EMR
95 */
96 emrManagedSlaveSecurityGroup?: string;
97 /**
98 * Additional security groups applied to the master node. Changing these
99 * replaces the cluster.
100 */
101 additionalMasterSecurityGroups?: string[];
102 /**
103 * Additional security groups applied to core/task nodes. Changing these
104 * replaces the cluster.
105 */
106 additionalSlaveSecurityGroups?: string[];
107 }
108
109 export interface ClusterProps {
110 /**
111 * Display name of the cluster. If omitted, a deterministic physical name is
112 * generated. Changing the name replaces the cluster.
113 */
114 clusterName?: string;
115 /**
116 * Amazon EMR release, e.g. `"emr-7.5.0"`. Changing the release replaces the
117 * cluster.
118 */
119 releaseLabel: string;
120 /**
121 * Applications installed on the cluster, e.g. `["Spark", "Hadoop"]`.
122 * Changing the set replaces the cluster.
123 */
124 applications?: string[];
125 /**
126 * EC2 instance topology (master/core instance groups, subnet, key pair,
127 * security groups).
128 */
129 instances?: ClusterInstancesConfig;
130 /**
131 * IAM service role EMR assumes to manage cluster resources, e.g. a role
132 * trusting `elasticmapreduce.amazonaws.com`. Changing the role replaces
133 * the cluster.
134 */
135 serviceRole: string;
136 /**
137 * IAM instance profile (also called the job-flow role or EC2 instance
138 * profile) assumed by the cluster's EC2 instances. Changing it replaces
139 * the cluster.
140 */
141 jobFlowRole: string;
142 /**
143 * S3 URI where cluster logs are written, e.g. `"s3://my-bucket/logs/"`.
144 * Changing it replaces the cluster.
145 */
146 logUri?: string;
147 /**
148 * Name of an EMR {@link SecurityConfiguration} applied at launch. Changing
149 * it replaces the cluster.
150 */
151 securityConfiguration?: string;
152 /**
153 * Application configuration overrides (raw EMR `Configuration` objects,
154 * e.g. `[{ Classification: "spark-defaults", Properties: {...} }]`).
155 * Changing them replaces the cluster.
156 */
157 configurations?: emr.Configuration[];
158 /**
159 * Whether all IAM principals in the account can see the cluster.
160 * Updateable in place.
161 * @default true
162 */
163 visibleToAllUsers?: boolean;
164 /**
165 * Size of the EBS root volume (GiB) of each instance. Changing it replaces
166 * the cluster.
167 */
168 ebsRootVolumeSize?: number;
169 /**
170 * Custom AMI ID for the cluster instances. Changing it replaces the
171 * cluster.
172 */
173 customAmiId?: string;
174 /**
175 * Number of steps that may execute concurrently (1-256). Updateable in
176 * place.
177 * @default 1
178 */
179 stepConcurrencyLevel?: number;
180 /**
181 * Auto-termination policy — the cluster terminates itself after being idle
182 * for the configured duration (60 seconds to 7 days). Updateable in place;
183 * removing the prop detaches the policy.
184 */
185 autoTerminationPolicy?: {
186 /**
187 * Idle time after which the cluster auto-terminates — e.g. `"1 hour"` or
188 * `Duration.hours(1)`. Sent to AWS as whole seconds.
189 */
190 idleTimeout: Duration.Input;
191 };
192 /**
193 * User-defined tags for the cluster.
194 */
195 tags?: Record<string, string>;
196 }
197
198 export interface Cluster extends Resource<
199 "AWS.EMR.Cluster",
200 ClusterProps,
201 {
202 /** The ID of the cluster (e.g. `j-2AXXXXXXGAPLF`). */
203 clusterId: string;
204 /** The ARN of the cluster. */
205 clusterArn: string;
206 /** The name of the cluster. */
207 clusterName: string;
208 /** The cluster state (e.g. `STARTING`, `RUNNING`, `WAITING`). */
209 state: string;
210 /** The public DNS name of the primary node, when reachable. */
211 masterPublicDnsName: string | undefined;
212 /** The tags applied to the cluster. */
213 tags: Record<string, string>;
214 },
215 never,
216 Providers
217 > {}
218
219 /**
220 * A provisioned Amazon EMR cluster (job flow) running open-source big-data
221 * frameworks such as Apache Spark and Hadoop on EC2 instances.
222 *
223 * Clusters take roughly 10-15 minutes to reach `WAITING` and bill per
224 * instance-hour while they exist. Each cluster needs an EMR service role and
225 * an EC2 instance profile (job-flow role); destroy clusters you are not
226 * using, or set `autoTerminationPolicy` as a safety net.
227 * ### Creating a Cluster
228 * **Example:** Spark Cluster in a Default-VPC Subnet
229 * ```typescript
230 * const cluster = yield* Cluster("Analytics", {
231 * releaseLabel: "emr-7.5.0",
232 * applications: ["Spark", "Hadoop"],
233 * serviceRole: serviceRole.roleName,
234 * jobFlowRole: instanceProfile.instanceProfileName,
235 * logUri: Output.interpolate`s3://${logsBucket.bucketName}/logs/`,
236 * instances: {
237 * masterInstanceType: "m5.xlarge",
238 * coreInstanceType: "m5.xlarge",
239 * coreInstanceCount: 1,
240 * ec2SubnetId: subnetId,
241 * },
242 * });
243 * ```
244 *
245 * **Example:** Cluster with an Auto-Termination Safety Net
246 * ```typescript
247 * const cluster = yield* Cluster("Batch", {
248 * releaseLabel: "emr-7.5.0",
249 * applications: ["Spark"],
250 * serviceRole: serviceRole.roleName,
251 * jobFlowRole: instanceProfile.instanceProfileName,
252 * autoTerminationPolicy: { idleTimeout: "1 hour" },
253 * stepConcurrencyLevel: 4,
254 * });
255 * ```
256 *
257 * ### Applying a Security Configuration
258 * **Example:** Cluster with Encryption Settings
259 * ```typescript
260 * const config = yield* SecurityConfiguration("Encryption", {
261 * securityConfiguration: {
262 * EncryptionConfiguration: {
263 * EnableInTransitEncryption: false,
264 * EnableAtRestEncryption: false,
265 * },
266 * },
267 * });
268 * const cluster = yield* Cluster("Secure", {
269 * releaseLabel: "emr-7.5.0",
270 * serviceRole: serviceRole.roleName,
271 * jobFlowRole: instanceProfile.instanceProfileName,
272 * securityConfiguration: config.securityConfigurationName,
273 * });
274 * ```
275 *
276 * @resource
277 */
278 export const Cluster = Resource<Cluster>("AWS.EMR.Cluster");
279
280 // States in which a cluster is (still) a live, converging resource.
281 const ACTIVE_STATES = ["STARTING", "BOOTSTRAPPING", "RUNNING", "WAITING"] as const;
282
283 // Terminal states: the cluster is gone or irreversibly going away.
284 const TERMINAL_STATES = new Set(["TERMINATING", "TERMINATED", "TERMINATED_WITH_ERRORS"]);
285
286 const toTagRecord = (
287 tags: ReadonlyArray<{ Key?: string; Value?: string }> | undefined,
288 ): Record<string, string> =>
289 Object.fromEntries(
290 (tags ?? [])
291 .filter(
292 (tag): tag is { Key: string; Value: string } =>
293 typeof tag.Key === "string" && typeof tag.Value === "string",
294 )
295 .map((tag) => [tag.Key, tag.Value]),
296 );
297
298 const sameStringSet = (
299 a: readonly string[] | undefined,
300 b: readonly string[] | undefined,
301 ): boolean => {
302 const left = [...(a ?? [])].sort();
303 const right = [...(b ?? [])].sort();
304 return left.length === right.length && left.every((v, i) => v === right[i]);
305 };
306
307 /** Build the master/core instance-group configuration for RunJobFlow. */
308 const buildInstanceGroups = (
309 instances: ClusterInstancesConfig | undefined,
310 ): emr.InstanceGroupConfig[] => {
311 const coreCount = instances?.coreInstanceCount ?? 1;
312 return [
313 {
314 Name: "Master",
315 InstanceRole: "MASTER",
316 InstanceType: instances?.masterInstanceType ?? "m5.xlarge",
317 InstanceCount: 1,
318 },
319 ...(coreCount > 0
320 ? [
321 {
322 Name: "Core",
323 InstanceRole: "CORE" as const,
324 InstanceType: instances?.coreInstanceType ?? "m5.xlarge",
325 InstanceCount: coreCount,
326 },
327 ]
328 : []),
329 ];
330 };
331
332 export const ClusterProvider = () =>
333 Provider.effect(
334 Cluster,
335 Effect.gen(function* () {
336 const toName = (id: string, props: Partial<ClusterProps>) =>
337 props.clusterName
338 ? Effect.succeed(props.clusterName)
339 : createPhysicalName({ id, maxLength: 80 });
340
341 // Describe a cluster by id; a terminal or missing cluster reads as
342 // undefined — a terminated EMR cluster cannot be revived or updated.
343 const readCluster = Effect.fn(function* (clusterId: string) {
344 const response = yield* emr
345 .describeCluster({ ClusterId: clusterId })
346 .pipe(Effect.catchTag("ClusterNotFound", () => Effect.undefined));
347 const cluster = response?.Cluster;
348 if (!cluster?.Id) return undefined;
349 if (TERMINAL_STATES.has(cluster.Status?.State ?? "")) return undefined;
350 return cluster;
351 });
352
353 // Cluster names are not unique in EMR; the deterministic physical name
354 // is. Find the live (non-terminal) cluster carrying our name — used
355 // when state was lost and we only know the derived name.
356 const findClusterByName = Effect.fn(function* (name: string) {
357 const summary = yield* emr.listClusters.items({ ClusterStates: [...ACTIVE_STATES] }).pipe(
358 Stream.filter((summary) => summary.Name === name),
359 Stream.runHead,
360 Effect.map(Option.getOrUndefined),
361 );
362 return summary?.Id ? yield* readCluster(summary.Id) : undefined;
363 });
364
365 // Bounded readiness wait. A Spark/Hadoop cluster typically reaches
366 // WAITING in 10-15 minutes; budget ~30 min (90 * 20s). A terminal
367 // state stops the wait immediately (non-retryable).
368 const waitForReady = Effect.fn(function* (clusterId: string) {
369 const policy = Schedule.max([Schedule.fixed("20 seconds"), Schedule.recurs(90)]);
370 return yield* emr.describeCluster({ ClusterId: clusterId }).pipe(
371 Effect.flatMap(
372 Effect.fnUntraced(function* (response) {
373 const cluster = response.Cluster;
374 const state = cluster?.Status?.State ?? "STARTING";
375 if (TERMINAL_STATES.has(state)) {
376 const reason = cluster?.Status?.StateChangeReason;
377 return yield* new EMRClusterOperationFailed({
378 message:
379 `EMR cluster '${clusterId}' reached terminal state '${state}'` +
380 ` (${reason?.Code ?? "unknown"}: ${reason?.Message ?? "no message"})`,
381 });
382 }
383 if (state !== "WAITING" && state !== "RUNNING") {
384 return yield* new EMRClusterNotReady({
385 message: `EMR cluster '${clusterId}' not ready (state: ${state})`,
386 });
387 }
388 return cluster!;
389 }),
390 ),
391 Effect.retry({
392 while: (e) => e instanceof Error && !e.message.includes("terminal state"),
393 schedule: policy,
394 }),
395 );
396 });
397
398 const toAttrs = Effect.fn(function* (cluster: emr.Cluster) {
399 if (!cluster.Id || !cluster.ClusterArn || !cluster.Name) {
400 return yield* new EMRClusterUnexpectedResponse({
401 message: `EMR cluster '${cluster.Id}' is missing its ARN or name`,
402 });
403 }
404 return {
405 clusterId: cluster.Id,
406 clusterArn: cluster.ClusterArn,
407 clusterName: cluster.Name,
408 state: cluster.Status?.State ?? "STARTING",
409 masterPublicDnsName: cluster.MasterPublicDnsName,
410 tags: toTagRecord(cluster.Tags),
411 };
412 });
413
414 return {
415 stables: ["clusterId", "clusterArn", "clusterName"],
416
417 diff: Effect.fn(function* ({ id, olds, news }) {
418 if (!isResolved(news)) return undefined;
419 const n = news;
420 const o = olds;
421 if (n === undefined || o === undefined) return undefined;
422 if ((yield* toName(id, o)) !== (yield* toName(id, n))) {
423 return { action: "replace" } as const;
424 }
425 // Create-only cluster-level properties force a replacement.
426 if (n.releaseLabel !== o.releaseLabel) {
427 return { action: "replace" } as const;
428 }
429 if (!sameStringSet(n.applications, o.applications)) {
430 return { action: "replace" } as const;
431 }
432 if (n.serviceRole !== o.serviceRole) {
433 return { action: "replace" } as const;
434 }
435 if (n.jobFlowRole !== o.jobFlowRole) {
436 return { action: "replace" } as const;
437 }
438 if ((n.logUri ?? undefined) !== (o.logUri ?? undefined)) {
439 return { action: "replace" } as const;
440 }
441 if ((n.securityConfiguration ?? undefined) !== (o.securityConfiguration ?? undefined)) {
442 return { action: "replace" } as const;
443 }
444 if (JSON.stringify(n.configurations ?? []) !== JSON.stringify(o.configurations ?? [])) {
445 return { action: "replace" } as const;
446 }
447 if ((n.ebsRootVolumeSize ?? undefined) !== (o.ebsRootVolumeSize ?? undefined)) {
448 return { action: "replace" } as const;
449 }
450 if ((n.customAmiId ?? undefined) !== (o.customAmiId ?? undefined)) {
451 return { action: "replace" } as const;
452 }
453 // Create-only instance topology.
454 const ni = n.instances ?? {};
455 const oi = o.instances ?? {};
456 if (
457 (ni.masterInstanceType ?? "m5.xlarge") !== (oi.masterInstanceType ?? "m5.xlarge") ||
458 (ni.coreInstanceType ?? "m5.xlarge") !== (oi.coreInstanceType ?? "m5.xlarge") ||
459 (ni.ec2SubnetId ?? undefined) !== (oi.ec2SubnetId ?? undefined) ||
460 (ni.ec2KeyName ?? undefined) !== (oi.ec2KeyName ?? undefined) ||
461 (ni.emrManagedMasterSecurityGroup ?? undefined) !==
462 (oi.emrManagedMasterSecurityGroup ?? undefined) ||
463 (ni.emrManagedSlaveSecurityGroup ?? undefined) !==
464 (oi.emrManagedSlaveSecurityGroup ?? undefined) ||
465 !sameStringSet(ni.additionalMasterSecurityGroups, oi.additionalMasterSecurityGroups) ||
466 !sameStringSet(ni.additionalSlaveSecurityGroups, oi.additionalSlaveSecurityGroups)
467 ) {
468 return { action: "replace" } as const;
469 }
470 // The core group can resize in place, but crossing the 0 boundary
471 // adds/removes the whole group — replacement.
472 const newCore = ni.coreInstanceCount ?? 1;
473 const oldCore = oi.coreInstanceCount ?? 1;
474 if ((newCore === 0) !== (oldCore === 0)) {
475 return { action: "replace" } as const;
476 }
477 }),
478
479 read: Effect.fn(function* ({ id, olds, output }) {
480 const cluster = output?.clusterId
481 ? yield* readCluster(output.clusterId)
482 : yield* findClusterByName(yield* toName(id, olds ?? {}));
483 if (!cluster) return undefined;
484 const attrs = yield* toAttrs(cluster);
485 return (yield* hasAlchemyTags(id, attrs.tags)) ? attrs : Unowned(attrs);
486 }),
487
488 reconcile: Effect.fn(function* ({ id, news, output, session }) {
489 const props = news!;
490 const name = output?.clusterName ?? (yield* toName(id, props));
491 const internalTags = yield* createInternalTags(id);
492 const desiredTags = { ...internalTags, ...props.tags };
493
494 // 1. Observe — cloud state is authoritative; output is only an id
495 // cache. A terminated cluster reads as missing.
496 let observed = output?.clusterId
497 ? yield* readCluster(output.clusterId)
498 : yield* findClusterByName(name);
499
500 // 2. Ensure — launch the job flow if missing. Create errors
501 // (invalid roles, subnet, release) propagate directly.
502 let clusterId = observed?.Id;
503 if (clusterId === undefined) {
504 const instances = props.instances ?? {};
505 const created = yield* emr
506 .runJobFlow({
507 Name: name,
508 ReleaseLabel: props.releaseLabel,
509 Applications: (props.applications ?? []).map((Name) => ({
510 Name,
511 })),
512 Instances: {
513 InstanceGroups: buildInstanceGroups(props.instances),
514 Ec2SubnetId: instances.ec2SubnetId,
515 Ec2KeyName: instances.ec2KeyName,
516 KeepJobFlowAliveWhenNoSteps: instances.keepJobFlowAliveWhenNoSteps ?? true,
517 TerminationProtected: instances.terminationProtected ?? false,
518 UnhealthyNodeReplacement: instances.unhealthyNodeReplacement,
519 EmrManagedMasterSecurityGroup: instances.emrManagedMasterSecurityGroup,
520 EmrManagedSlaveSecurityGroup: instances.emrManagedSlaveSecurityGroup,
521 AdditionalMasterSecurityGroups: instances.additionalMasterSecurityGroups,
522 AdditionalSlaveSecurityGroups: instances.additionalSlaveSecurityGroups,
523 },
524 ServiceRole: props.serviceRole,
525 JobFlowRole: props.jobFlowRole,
526 LogUri: props.logUri,
527 SecurityConfiguration: props.securityConfiguration,
528 Configurations: props.configurations,
529 VisibleToAllUsers: props.visibleToAllUsers ?? true,
530 EbsRootVolumeSize: props.ebsRootVolumeSize,
531 CustomAmiId: props.customAmiId,
532 StepConcurrencyLevel: props.stepConcurrencyLevel,
533 AutoTerminationPolicy: props.autoTerminationPolicy
534 ? {
535 IdleTimeout: toWireSeconds(props.autoTerminationPolicy.idleTimeout),
536 }
537 : undefined,
538 Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
539 Key,
540 Value,
541 })),
542 })
543 .pipe(
544 // A freshly created EC2 instance profile is not yet visible to
545 // EMR (IAM eventual consistency) — RunJobFlow rejects it with
546 // "Invalid InstanceProfile" for several seconds. Retry bounded.
547 Effect.retry({
548 while: (e) => e._tag === "EmrInstanceProfileNotReady",
549 schedule: Schedule.spaced("5 seconds"),
550 times: 10,
551 }),
552 );
553 if (!created.JobFlowId) {
554 return yield* new EMRClusterUnexpectedResponse({
555 message: `RunJobFlow for '${name}' returned no JobFlowId`,
556 });
557 }
558 clusterId = created.JobFlowId;
559 }
560
561 // Provisioning surfaces as STARTING/BOOTSTRAPPING; wait (bounded)
562 // for WAITING/RUNNING so modify calls don't race bootstrap.
563 observed = yield* waitForReady(clusterId);
564
565 // 3. Sync — per mutable aspect, diff OBSERVED state against the
566 // desired props and apply only the delta.
567 const desiredVisible = props.visibleToAllUsers ?? true;
568 if ((observed.VisibleToAllUsers ?? true) !== desiredVisible) {
569 yield* emr.setVisibleToAllUsers({
570 JobFlowIds: [clusterId],
571 VisibleToAllUsers: desiredVisible,
572 });
573 }
574
575 const desiredKeepAlive = props.instances?.keepJobFlowAliveWhenNoSteps ?? true;
576 const observedKeepAlive = !(observed.AutoTerminate ?? false);
577 if (observedKeepAlive !== desiredKeepAlive) {
578 yield* emr.setKeepJobFlowAliveWhenNoSteps({
579 JobFlowIds: [clusterId],
580 KeepJobFlowAliveWhenNoSteps: desiredKeepAlive,
581 });
582 }
583
584 const desiredProtected = props.instances?.terminationProtected;
585 if (
586 desiredProtected !== undefined &&
587 (observed.TerminationProtected ?? false) !== desiredProtected
588 ) {
589 yield* emr.setTerminationProtection({
590 JobFlowIds: [clusterId],
591 TerminationProtected: desiredProtected,
592 });
593 }
594
595 const desiredUnhealthy = props.instances?.unhealthyNodeReplacement;
596 if (
597 desiredUnhealthy !== undefined &&
598 (observed.UnhealthyNodeReplacement ?? false) !== desiredUnhealthy
599 ) {
600 yield* emr.setUnhealthyNodeReplacement({
601 JobFlowIds: [clusterId],
602 UnhealthyNodeReplacement: desiredUnhealthy,
603 });
604 }
605
606 if (
607 props.stepConcurrencyLevel !== undefined &&
608 (observed.StepConcurrencyLevel ?? 1) !== props.stepConcurrencyLevel
609 ) {
610 yield* emr.modifyCluster({
611 ClusterId: clusterId,
612 StepConcurrencyLevel: props.stepConcurrencyLevel,
613 });
614 }
615
616 // Auto-termination policy: observe, then put/remove the delta.
617 const observedPolicy = yield* emr.getAutoTerminationPolicy({ ClusterId: clusterId }).pipe(
618 Effect.map((r) => r.AutoTerminationPolicy),
619 Effect.orElseSucceed(() => undefined),
620 );
621 if (props.autoTerminationPolicy !== undefined) {
622 const desiredIdleTimeout = toWireSeconds(props.autoTerminationPolicy.idleTimeout);
623 if (observedPolicy?.IdleTimeout !== desiredIdleTimeout) {
624 yield* emr.putAutoTerminationPolicy({
625 ClusterId: clusterId,
626 AutoTerminationPolicy: {
627 IdleTimeout: desiredIdleTimeout,
628 },
629 });
630 }
631 } else if (observedPolicy !== undefined) {
632 yield* emr.removeAutoTerminationPolicy({ ClusterId: clusterId });
633 }
634
635 // Core instance-group resize (in place between non-zero counts).
636 const desiredCore = props.instances?.coreInstanceCount ?? 1;
637 if (desiredCore > 0) {
638 const core = yield* emr.listInstanceGroups.items({ ClusterId: clusterId }).pipe(
639 Stream.filter((g) => g.InstanceGroupType === "CORE"),
640 Stream.runHead,
641 Effect.map(Option.getOrUndefined),
642 );
643 if (core?.Id !== undefined && (core.RequestedInstanceCount ?? 0) !== desiredCore) {
644 yield* emr.modifyInstanceGroups({
645 ClusterId: clusterId,
646 InstanceGroups: [{ InstanceGroupId: core.Id, InstanceCount: desiredCore }],
647 });
648 }
649 }
650
651 // 3b. Sync tags — diff against OBSERVED cloud tags.
652 const observedTags = toTagRecord(observed.Tags);
653 const { removed, upsert } = diffTags(observedTags, desiredTags);
654 if (upsert.length > 0) {
655 yield* emr.addTags({ ResourceId: clusterId, Tags: upsert });
656 }
657 if (removed.length > 0) {
658 yield* emr.removeTags({ ResourceId: clusterId, TagKeys: removed });
659 }
660
661 // 4. Return fresh attributes.
662 const final = yield* readCluster(clusterId);
663 yield* session.note(clusterId);
664 return yield* toAttrs(final ?? observed);
665 }),
666
667 delete: Effect.fn(function* ({ output }) {
668 const clusterId = output.clusterId;
669 // A cluster that is already terminal (or gone) is success.
670 const observed = yield* readCluster(clusterId);
671 if (observed === undefined) return;
672 // Lift termination protection so TerminateJobFlows succeeds.
673 if (observed.TerminationProtected === true) {
674 yield* emr
675 .setTerminationProtection({
676 JobFlowIds: [clusterId],
677 TerminationProtected: false,
678 })
679 .pipe(Effect.catchTag("JobFlowNotFound", () => Effect.void));
680 }
681 yield* emr
682 .terminateJobFlows({ JobFlowIds: [clusterId] })
683 .pipe(Effect.catchTag("JobFlowNotFound", () => Effect.void));
684 // Termination is irreversible once initiated; wait (bounded) for
685 // the state to leave the active set. Full instance teardown takes
686 // 5-10 more minutes server-side and needs no babysitting.
687 yield* emr.describeCluster({ ClusterId: clusterId }).pipe(
688 Effect.flatMap((response) => {
689 const state = response.Cluster?.Status?.State ?? "TERMINATED";
690 return TERMINAL_STATES.has(state)
691 ? Effect.void
692 : Effect.fail(
693 new EMRClusterNotReady({
694 message: `EMR cluster '${clusterId}' still active (state: ${state})`,
695 }),
696 );
697 }),
698 Effect.catchTag("ClusterNotFound", () => Effect.void),
699 Effect.retry({
700 schedule: Schedule.max([Schedule.fixed("10 seconds"), Schedule.recurs(30)]),
701 }),
702 );
703 }),
704
705 list: () =>
706 emr.listClusters.items({ ClusterStates: [...ACTIVE_STATES] }).pipe(
707 Stream.runCollect,
708 Effect.map((chunk) =>
709 Array.from(chunk).flatMap((summary) => (summary.Id ? [summary.Id] : [])),
710 ),
711 Effect.flatMap(
712 Effect.forEach((clusterId) => readCluster(clusterId), {
713 concurrency: 4,
714 }),
715 ),
716 Effect.map((clusters) =>
717 clusters.flatMap((cluster) => (cluster === undefined ? [] : [cluster])),
718 ),
719 Effect.flatMap(
720 Effect.forEach((cluster) => toAttrs(cluster), {
721 concurrency: 4,
722 }),
723 ),
724 ),
725 };
726 }),
727 );