The best way to control Claude Code, Codex and more from any computer or phone

main
ts 761 lines 28.3 KB
Raw
1 import * as sagemaker from "@distilled.cloud/aws/sagemaker";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import type { Input } from "alchemy/Input";
5 import * as Output from "alchemy/Output";
6 import { createPhysicalName } from "alchemy/PhysicalName";
7 import * as Provider from "alchemy/Provider";
8 import { Resource } from "alchemy/Resource";
9 import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
10 import * as Data from "effect/Data";
11 import * as Effect from "effect/Effect";
12 import * as Schedule from "effect/Schedule";
13 import * as EffectStream from "effect/Stream";
14 import type { Providers } from "../Providers.ts";
15
16 export class SageMakerClusterInvalidProps extends Data.TaggedError("SageMakerClusterInvalidProps")<{
17 readonly message: string;
18 readonly cause?: unknown;
19 }> {}
20
21 export class SageMakerClusterUnexpectedResponse extends Data.TaggedError(
22 "SageMakerClusterUnexpectedResponse",
23 )<{
24 readonly message: string;
25 readonly cause?: unknown;
26 }> {}
27
28 export type ClusterStatus = sagemaker.ClusterStatus;
29
30 /**
31 * The well-known node label carrying the HyperPod instance-group name.
32 * HyperPod nodes join the orchestrating EKS cluster as ordinary Kubernetes
33 * nodes carrying this label.
34 */
35 export const HYPERPOD_INSTANCE_GROUP_LABEL = "sagemaker.amazonaws.com/instance-group-name";
36
37 /** The well-known node label carrying HyperPod's node health verdict. */
38 export const HYPERPOD_NODE_HEALTH_LABEL = "sagemaker.amazonaws.com/node-health-status";
39
40 /**
41 * An instance group surfaced on the cluster's attributes — Kubernetes
42 * workloads pin themselves to the group by referencing its
43 * {@link ClusterInstanceGroupRef.nodeSelector}.
44 */
45 export interface ClusterInstanceGroupRef {
46 /** The group's name (the `sagemaker.amazonaws.com/instance-group-name` node label). */
47 InstanceGroupName: string;
48 /** The group's instance type (e.g. `ml.g5.xlarge`). */
49 InstanceType: string | undefined;
50 /** Instances currently in service. */
51 CurrentCount: number | undefined;
52 /** Instances the group is converging toward. */
53 TargetCount: number | undefined;
54 /**
55 * A Kubernetes node selector pinning pods onto this group's
56 * health-checked nodes — pass it to a `Kubernetes.Job` /
57 * `Kubernetes.Deployment` through the `podTemplate` escape hatch:
58 *
59 * ```ts
60 * podTemplate: {
61 * spec: { nodeSelector: hyperpod.instanceGroups.workers.nodeSelector },
62 * }
63 * ```
64 */
65 nodeSelector: Record<string, string>;
66 }
67
68 /**
69 * One instance group's specification, sans name — the group's name is the
70 * key in {@link ClusterProps.instanceGroups}.
71 */
72 export type ClusterInstanceGroup = Omit<
73 sagemaker.ClusterInstanceGroupSpecification,
74 "InstanceGroupName"
75 >;
76
77 /**
78 * One restricted instance group's specification, sans name — the group's
79 * name is the key in {@link ClusterProps.restrictedInstanceGroups}.
80 */
81 export type ClusterRestrictedInstanceGroup = Omit<
82 sagemaker.ClusterRestrictedInstanceGroupSpecification,
83 "InstanceGroupName"
84 >;
85
86 export interface ClusterProps {
87 /**
88 * Name of the HyperPod cluster. Maximum 63 characters, alphanumeric and
89 * hyphens.
90 * @default ${app}-${stage}-${id}
91 */
92 clusterName?: string;
93 /**
94 * The instance groups of the cluster, keyed by group name. Each group
95 * specifies its instance type, instance count, lifecycle config (S3 URI
96 * + `on_create` script) and execution role.
97 *
98 * Groups are updated in place; removing a key deletes the group from the
99 * cluster. The keys carry through to the cluster's `instanceGroups`
100 * attribute, so `hyperpod.instanceGroups.workers` is typed per key.
101 */
102 instanceGroups?: Record<string, ClusterInstanceGroup>;
103 /**
104 * Restricted instance groups for HyperPod clusters running Amazon-managed
105 * workloads (e.g. Nova model customization), keyed by group name.
106 */
107 restrictedInstanceGroups?: Record<string, ClusterRestrictedInstanceGroup>;
108 /**
109 * Shared environment configuration (e.g. FSx for Lustre) for restricted
110 * instance groups.
111 */
112 restrictedInstanceGroupsConfig?: sagemaker.ClusterRestrictedInstanceGroupsConfig;
113 /**
114 * VPC to launch the cluster's nodes into. Changing the VPC replaces the
115 * cluster.
116 */
117 vpcConfig?: sagemaker.VpcConfig;
118 /**
119 * The cluster's orchestrator. Set `{ Eks: { ClusterArn } }` to attach the
120 * HyperPod cluster to an EKS control plane; defaults to Slurm when
121 * omitted.
122 */
123 orchestrator?: sagemaker.ClusterOrchestrator;
124 /**
125 * Whether HyperPod automatically replaces faulty nodes.
126 * @default "Automatic"
127 */
128 nodeRecovery?: sagemaker.ClusterNodeRecovery;
129 /**
130 * Managed tiered storage (memory + local NVMe) configuration for
131 * checkpointing.
132 */
133 tieredStorageConfig?: sagemaker.ClusterTieredStorageConfig;
134 /**
135 * Set to `"Continuous"` to enable continuous node provisioning, where
136 * HyperPod provisions capacity as it becomes available instead of
137 * all-or-nothing.
138 */
139 nodeProvisioningMode?: sagemaker.ClusterNodeProvisioningMode;
140 /**
141 * ARN of the IAM role HyperPod assumes for cluster-level operations
142 * (required for continuous provisioning and autoscaling).
143 */
144 clusterRole?: string;
145 /**
146 * Cluster autoscaling configuration (e.g. Karpenter) for EKS-orchestrated
147 * clusters.
148 */
149 autoScaling?: sagemaker.ClusterAutoScalingConfig;
150 /**
151 * Tags to associate with the cluster. Merged with internal Alchemy tags.
152 */
153 tags?: Record<string, string>;
154 }
155
156 export interface Cluster extends Resource<
157 "AWS.SageMaker.Cluster",
158 ClusterProps,
159 {
160 /**
161 * The cluster's name.
162 */
163 clusterName: string;
164 /**
165 * ARN of the cluster.
166 */
167 clusterArn: string;
168 /**
169 * The cluster's status (`InService` once reconciled).
170 */
171 clusterStatus: ClusterStatus;
172 /**
173 * ARN of the orchestrating EKS cluster, when EKS-orchestrated.
174 */
175 orchestratorEksClusterArn: string | undefined;
176 /**
177 * The cluster's instance groups, keyed by group name. Pin a
178 * Kubernetes workload onto a group through the resource graph with
179 * its node selector:
180 * `hyperpod.instanceGroups.workers.nodeSelector`.
181 */
182 instanceGroups: Record<string, ClusterInstanceGroupRef>;
183 },
184 never,
185 Providers
186 > {}
187
188 const ClusterResource = Resource<Cluster>("AWS.SageMaker.Cluster");
189
190 /**
191 * A `Cluster` narrowed to the instance-group keys declared in its props —
192 * `hyperpod.instanceGroups.workers` is typed per key, and a typo'd key is
193 * a compile error.
194 */
195 export type ClusterOf<Groups> = Omit<Cluster, "instanceGroups"> & {
196 instanceGroups: Output.ObjectExpr<{ [K in keyof Groups]: ClusterInstanceGroupRef }, never>;
197 };
198
199 /**
200 * An Amazon SageMaker HyperPod cluster — a resilient, persistent cluster of
201 * ML compute for distributed training and inference, orchestrated by Slurm
202 * or EKS, with automatic faulty-node recovery and deep health checks.
203 *
204 * Provisioning a HyperPod cluster takes 10–25 minutes; instance groups are
205 * updated in place and removing a group from `instanceGroups` deletes it
206 * from the cluster.
207 * ### Creating Clusters
208 * **Example:** Slurm-Orchestrated Cluster
209 * ```typescript
210 * import * as AWS from "@alchemy.run/aws";
211 *
212 * const cluster = yield* AWS.SageMaker.Cluster("TrainingCluster", {
213 * instanceGroups: {
214 * controller: {
215 * InstanceType: "ml.t3.medium",
216 * InstanceCount: 1,
217 * ExecutionRole: role.roleArn,
218 * // HyperPod now provisions Slurm clusters in Continuous mode, which
219 * // requires a Slurm node type on at least one instance group.
220 * SlurmConfig: { NodeType: "Controller" },
221 * LifeCycleConfig: {
222 * SourceS3Uri: `s3://${bucket.bucketName}/lifecycle`,
223 * OnCreate: "on_create.sh",
224 * },
225 * },
226 * },
227 * });
228 * ```
229 *
230 * **Example:** EKS-Orchestrated Cluster
231 * ```typescript
232 * // The EKS cluster must use the `API` (or `API_AND_CONFIG_MAP`)
233 * // authentication mode — pass `accessConfig` explicitly, EKS's own
234 * // CONFIG_MAP default is rejected. LifeCycleConfig is required for
235 * // EKS-orchestrated instance groups too.
236 * const hyperpod = yield* AWS.SageMaker.Cluster("EksHyperPod", {
237 * orchestrator: { Eks: { ClusterArn: eksCluster.clusterArn } },
238 * vpcConfig: {
239 * SecurityGroupIds: [securityGroupId],
240 * Subnets: network.privateSubnetIds,
241 * },
242 * instanceGroups: {
243 * workers: {
244 * InstanceType: "ml.g5.xlarge",
245 * InstanceCount: 2,
246 * ExecutionRole: role.roleArn,
247 * LifeCycleConfig: {
248 * SourceS3Uri: `s3://${bucket.bucketName}/lifecycle`,
249 * OnCreate: "on_create.sh",
250 * },
251 * },
252 * },
253 * nodeRecovery: "Automatic",
254 * });
255 *
256 * // The keys carry through to the attributes — typed per key:
257 * const workers = hyperpod.instanceGroups.workers;
258 * ```
259 *
260 * ### Running Workloads (Slurm)
261 * **Example:** Submit jobs from the login node over SSM
262 * ```sh
263 * # Slurm jobs are submitted on the cluster itself. Each node is an SSM
264 * # target named sagemaker-cluster:<cluster-id>_<instance-group>-<instance-id>
265 * # (list nodes with `aws sagemaker list-cluster-nodes`).
266 * aws ssm start-session \
267 * --target sagemaker-cluster:6wl4at0i68c6_controller-i-0123456789abcdef0
268 * # then, on the node:
269 * sbatch --nodes=4 train.sbatch
270 * ```
271 *
272 * ### Running Workloads (EKS)
273 * **Example:** Low level: apply any Kubernetes manifest to the orchestrator
274 * ```typescript
275 * // HyperPod nodes are ordinary EKS nodes — target them from a raw
276 * // manifest (a PyTorchJob CRD, a batch/v1 Job, ...) with the well-known
277 * // node labels.
278 * const job = yield* AWS.EKS.Manifest("RawTrainJob", {
279 * cluster: eksCluster,
280 * manifest: {
281 * apiVersion: "batch/v1",
282 * kind: "Job",
283 * metadata: { name: "raw-train", namespace: "default" },
284 * spec: {
285 * template: {
286 * spec: {
287 * nodeSelector: {
288 * "sagemaker.amazonaws.com/node-health-status": "Schedulable",
289 * "sagemaker.amazonaws.com/instance-group-name": "workers",
290 * },
291 * containers: [{ name: "train", image: "ghcr.io/acme/train:v3" }],
292 * restartPolicy: "Never",
293 * },
294 * },
295 * },
296 * },
297 * });
298 * ```
299 *
300 * **Example:** High level: an effectful Job pinned to HyperPod nodes
301 * ```typescript
302 * // Kubernetes.Job / Kubernetes.Deployment run on HyperPod via the
303 * // orchestrating EKS cluster in plain Kubernetes vocabulary — the
304 * // HyperPod resources expose the derived values as attributes: the
305 * // group's `nodeSelector`, the quota's governed `namespace` and Kueue
306 * // `queueName`.
307 * const evaluate = yield* Kubernetes.Job(
308 * "Evaluate",
309 * {
310 * cluster: eksCluster,
311 * main: import.meta.url,
312 * namespace: quota.namespace,
313 * labels: {
314 * [AWS.SageMaker.KUEUE_QUEUE_NAME_LABEL]: quota.queueName,
315 * [AWS.SageMaker.KUEUE_PRIORITY_CLASS_LABEL]: "training-priority",
316 * },
317 * podTemplate: {
318 * spec: {
319 * nodeSelector: hyperpod.instanceGroups.workers.nodeSelector,
320 * },
321 * },
322 * },
323 * Effect.gen(function* () {
324 * const putItem = yield* AWS.DynamoDB.PutItem(resultsTable);
325 * return {
326 * run: Effect.gen(function* () {
327 * // evaluation logic; bindings land IAM on the pod-identity role
328 * }),
329 * };
330 * }).pipe(Effect.provide(AWS.DynamoDB.PutItemHttp)),
331 * );
332 * ```
333 *
334 * ### Task Governance
335 * **Example:** Prioritize workloads with a scheduler policy and team quotas
336 * ```typescript
337 * // Requires the amazon-sagemaker-hyperpod-taskgovernance EKS add-on.
338 * const policy = yield* AWS.SageMaker.ClusterSchedulerConfig("Scheduler", {
339 * clusterArn: hyperpod.clusterArn,
340 * schedulerConfig: {
341 * PriorityClasses: [{ Name: "training", Weight: 90 }],
342 * FairShare: "Enabled",
343 * },
344 * });
345 *
346 * // Creates the hyperpod-ns-research namespace + Kueue LocalQueue —
347 * // exposed as `quota.namespace` / `quota.queueName` for governed
348 * // Kubernetes workloads to reference.
349 * const quota = yield* AWS.SageMaker.ComputeQuota("ResearchQuota", {
350 * clusterArn: hyperpod.clusterArn,
351 * computeQuotaTarget: { TeamName: "research", FairShareWeight: 10 },
352 * computeQuotaConfig: {
353 * ComputeQuotaResources: [
354 * { InstanceType: "ml.g5.xlarge", Count: 1 },
355 * ],
356 * },
357 * });
358 * ```
359 *
360 * @resource
361 */
362 export const Cluster: {
363 <
364 const Props extends {
365 [prop in keyof ClusterProps]: Input<ClusterProps[prop]>;
366 },
367 >(
368 id: string,
369 props: Props | Effect.Effect<Props>,
370 ): Effect.Effect<ClusterOf<NonNullable<Props["instanceGroups"]>>, never, Providers>;
371 } & typeof ClusterResource = ClusterResource as never;
372
373 const createClusterName = (id: string, props: { clusterName?: string | undefined }) =>
374 props.clusterName ? Effect.succeed(props.clusterName) : createPhysicalName({ id, maxLength: 63 });
375
376 const describeClusterOrUndefined = (name: string) =>
377 sagemaker
378 .describeCluster({ ClusterName: name })
379 .pipe(Effect.catchTag("ResourceNotFound", () => Effect.undefined));
380
381 const fetchClusterTags = Effect.fn(function* (arn: string) {
382 const response = yield* sagemaker
383 .listTags({ ResourceArn: arn })
384 .pipe(Effect.catchTag("AccessDeniedException", () => Effect.undefined));
385 return Object.fromEntries(
386 (response?.Tags ?? []).flatMap((tag) =>
387 tag.Key !== undefined ? [[tag.Key, tag.Value ?? ""]] : [],
388 ),
389 );
390 });
391
392 const toAttrs = (described: sagemaker.DescribeClusterResponse): Cluster["Attributes"] => ({
393 clusterName: described.ClusterName ?? "",
394 clusterArn: described.ClusterArn,
395 clusterStatus: described.ClusterStatus,
396 orchestratorEksClusterArn: described.Orchestrator?.Eks?.ClusterArn,
397 instanceGroups: Object.fromEntries(
398 (described.InstanceGroups ?? []).flatMap((group) =>
399 group.InstanceGroupName !== undefined
400 ? [
401 [
402 group.InstanceGroupName,
403 {
404 InstanceGroupName: group.InstanceGroupName,
405 InstanceType: group.InstanceType,
406 CurrentCount: group.CurrentCount,
407 TargetCount: group.TargetCount,
408 nodeSelector: {
409 [HYPERPOD_NODE_HEALTH_LABEL]: "Schedulable",
410 [HYPERPOD_INSTANCE_GROUP_LABEL]: group.InstanceGroupName,
411 },
412 },
413 ],
414 ]
415 : [],
416 ),
417 ),
418 });
419
420 /**
421 * The cluster is still transitioning toward the awaited state — retried by
422 * the bounded wait schedule.
423 */
424 class ClusterNotReady extends Data.TaggedError("ClusterNotReady")<{
425 readonly clusterName: string;
426 readonly status: string | undefined;
427 }> {}
428
429 /**
430 * The cluster converged to the terminal `Failed` status.
431 */
432 export class ClusterFailed extends Data.TaggedError("ClusterFailed")<{
433 readonly clusterName: string;
434 readonly message: string | undefined;
435 }> {}
436
437 /**
438 * The cluster is stuck in `Deleting` because SageMaker's internal teardown
439 * cannot proceed — e.g. the instance group's execution role (which HyperPod
440 * assumes to delete node ENIs) was deleted or lost its
441 * `ec2:DeleteNetworkInterface` / `ec2:DeleteNetworkInterfacePermission`
442 * permissions mid-teardown. Retrying `deleteCluster` can never fix this;
443 * the role and its permissions must be restored out-of-band, after which
444 * SageMaker's own retry completes the deletion within a minute.
445 */
446 export class ClusterTeardownBlocked extends Data.TaggedError("ClusterTeardownBlocked")<{
447 readonly clusterName: string;
448 readonly message: string;
449 }> {}
450
451 // Explicitly-typed retry wrapper — an inline `Effect.retry` in provider
452 // lifecycle code leaks `Retry.Return`'s conditional type into declaration
453 // emit and widens the provider layer to `unknown` for every consumer of
454 // `AWS.providers()`.
455 const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>(
456 self: Effect.Effect<A, E, R>,
457 ): Effect.Effect<A, E, R> =>
458 Effect.retry(self, {
459 while: (e) => e._tag === "ClusterNotReady",
460 // HyperPod cluster provisioning routinely takes 10–25 minutes.
461 // Poll 15s up to ~35 min.
462 schedule: Schedule.max([Schedule.spaced("15 seconds"), Schedule.recurs(140)]),
463 });
464
465 // A freshly created execution role isn't assumable by SageMaker for a few
466 // seconds (IAM propagation) — createCluster rejects it with a
467 // ValidationException until it is.
468 const retryWhileRoleUnassumable = <
469 A,
470 E extends { readonly _tag: string; readonly message?: string },
471 R,
472 >(
473 self: Effect.Effect<A, E, R>,
474 ): Effect.Effect<A, E, R> =>
475 Effect.retry(self, {
476 while: (e): boolean =>
477 e._tag === "ValidationException" &&
478 (e.message?.includes("cannot assume the execution role") ?? false),
479 schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(12)]),
480 });
481
482 const waitForCluster = (name: string, target: "InService" | "Gone") =>
483 retryWhileNotReady(
484 Effect.gen(function* () {
485 const described = yield* describeClusterOrUndefined(name);
486 if (target === "Gone") {
487 if (described === undefined) return;
488 // SageMaker's internal teardown deletes node ENIs by assuming the
489 // instance group's execution role. If that role (or its EC2
490 // permissions) was deleted mid-teardown, every internal retry
491 // fails and the cluster stays `Deleting` forever — surface the
492 // node's permission failure immediately instead of polling the
493 // full wait budget on a delete that can never finish.
494 const nodes = yield* sagemaker
495 .listClusterNodes({ ClusterName: name })
496 .pipe(Effect.catchTag("ResourceNotFound", () => Effect.undefined));
497 const blocked = (nodes?.ClusterNodeSummaries ?? []).find((node) =>
498 node.InstanceStatus?.Message?.includes("does not have permission to perform"),
499 );
500 if (blocked !== undefined) {
501 return yield* new ClusterTeardownBlocked({
502 clusterName: name,
503 message: blocked.InstanceStatus?.Message ?? "",
504 });
505 }
506 return yield* new ClusterNotReady({
507 clusterName: name,
508 status: described.ClusterStatus,
509 });
510 }
511 if (described?.ClusterStatus === "InService") return;
512 if (described?.ClusterStatus === "Failed") {
513 return yield* new ClusterFailed({
514 clusterName: name,
515 message: described.FailureMessage,
516 });
517 }
518 return yield* new ClusterNotReady({
519 clusterName: name,
520 status: described?.ClusterStatus,
521 });
522 }),
523 );
524
525 /**
526 * The subset of an instance group's specification that `updateCluster`
527 * can change and `describeCluster` reports back — used to diff desired
528 * groups against observed groups.
529 */
530 const observedGroupIdentity = (
531 group: sagemaker.DescribeClusterResponse["InstanceGroups"][number],
532 ) => ({
533 count: group.TargetCount ?? group.CurrentCount,
534 type: group.InstanceType,
535 });
536
537 const desiredGroupIdentity = (group: ClusterInstanceGroup) => ({
538 count: group.InstanceCount,
539 type: group.InstanceType,
540 });
541
542 /** Fold the keyed props form into the API's named-array form. */
543 const toGroupSpecs = <G extends { InstanceGroupName?: string }>(
544 groups: Record<string, Omit<G, "InstanceGroupName">> | undefined,
545 ): G[] | undefined =>
546 groups === undefined
547 ? undefined
548 : Object.entries(groups).map(([name, group]) => ({ InstanceGroupName: name, ...group }) as G);
549
550 export const ClusterProvider = () =>
551 Provider.effect(
552 Cluster,
553 Effect.gen(function* () {
554 return {
555 stables: ["clusterName", "clusterArn"],
556 // HyperPod's internal teardown deletes node ENIs by assuming the
557 // instance group's execution role inside the cluster's VPC —
558 // deleting the role or network mid-teardown wedges the cluster in
559 // `Deleting` permanently, so nuke must fully delete clusters
560 // before touching these types.
561 nuke: {
562 dependsOn: [
563 "AWS.IAM.Role",
564 "AWS.IAM.Policy",
565 "AWS.IAM.InstanceProfile",
566 "AWS.EC2.Vpc",
567 "AWS.EC2.Subnet",
568 "AWS.EC2.SecurityGroup",
569 ],
570 },
571 list: () =>
572 Effect.gen(function* () {
573 const summaries = yield* sagemaker.listClusters.pages({}).pipe(
574 EffectStream.runCollect,
575 Effect.map((chunk) =>
576 Array.from(chunk).flatMap((page) => page.ClusterSummaries ?? []),
577 ),
578 );
579 return summaries.flatMap((s) =>
580 s.ClusterName !== undefined && s.ClusterArn !== undefined
581 ? [
582 {
583 clusterName: s.ClusterName,
584 clusterArn: s.ClusterArn,
585 clusterStatus: s.ClusterStatus ?? "InService",
586 orchestratorEksClusterArn: undefined,
587 instanceGroups: {},
588 },
589 ]
590 : [],
591 );
592 }),
593 read: Effect.fn(function* ({ id, olds, output }) {
594 const name = output?.clusterName ?? (yield* createClusterName(id, olds ?? {}));
595 const described = yield* describeClusterOrUndefined(name);
596 if (!described || described.ClusterStatus === "Deleting") {
597 return undefined;
598 }
599 const attrs = toAttrs(described);
600 const tags = yield* fetchClusterTags(attrs.clusterArn);
601 return (yield* hasAlchemyTags(id, tags as Tags)) ? attrs : Unowned(attrs);
602 }),
603 diff: Effect.fn(function* ({ id, news, olds }) {
604 if (!isResolved(news)) return;
605 if (olds === undefined) return;
606 const oldName = yield* createClusterName(id, olds);
607 const newName = yield* createClusterName(id, news);
608 // The VPC is fixed at creation — changing it (or the name)
609 // replaces the cluster. Everything else updates in place.
610 if (
611 oldName !== newName ||
612 JSON.stringify(olds.vpcConfig) !== JSON.stringify(news.vpcConfig)
613 ) {
614 return { action: "replace" } as const;
615 }
616 }),
617 reconcile: Effect.fn(function* ({ id, news, output, session }) {
618 if (!news) {
619 return yield* new SageMakerClusterInvalidProps({
620 message: "SageMaker HyperPod Cluster requires props",
621 });
622 }
623 const name = output?.clusterName ?? (yield* createClusterName(id, news));
624 const internalTags = yield* createInternalTags(id);
625 const desiredTags = { ...internalTags, ...news.tags };
626
627 // Observe.
628 let described = yield* describeClusterOrUndefined(name);
629
630 // Ensure — create if missing; tolerate the already-exists race.
631 if (described === undefined) {
632 yield* sagemaker
633 .createCluster({
634 ClusterName: name,
635 InstanceGroups: toGroupSpecs(news.instanceGroups),
636 RestrictedInstanceGroups: toGroupSpecs(news.restrictedInstanceGroups),
637 RestrictedInstanceGroupsConfig: news.restrictedInstanceGroupsConfig,
638 VpcConfig: news.vpcConfig,
639 Orchestrator: news.orchestrator,
640 NodeRecovery: news.nodeRecovery,
641 TieredStorageConfig: news.tieredStorageConfig,
642 NodeProvisioningMode: news.nodeProvisioningMode,
643 ClusterRole: news.clusterRole,
644 AutoScaling: news.autoScaling,
645 Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
646 Key,
647 Value,
648 })),
649 })
650 .pipe(
651 retryWhileRoleUnassumable,
652 Effect.catchTag("ResourceInUse", () => Effect.void),
653 );
654 yield* session.note(`Creating HyperPod cluster ${name} (typically 10-25 minutes)...`);
655 } else {
656 // Sync — diff observed instance groups + mutable settings
657 // against desired and apply one update call for the delta.
658 const observedGroups = new Map(
659 (described.InstanceGroups ?? []).flatMap((g) =>
660 g.InstanceGroupName !== undefined
661 ? [[g.InstanceGroupName, observedGroupIdentity(g)]]
662 : [],
663 ),
664 );
665 const desiredGroups = new Map(
666 Object.entries(news.instanceGroups ?? {}).map(([groupName, g]) => [
667 groupName,
668 desiredGroupIdentity(g),
669 ]),
670 );
671 const groupsToDelete = [...observedGroups.keys()].filter(
672 (groupName) => !desiredGroups.has(groupName),
673 );
674 const groupsChanged =
675 groupsToDelete.length > 0 ||
676 [...desiredGroups].some(
677 ([groupName, identity]) =>
678 JSON.stringify(observedGroups.get(groupName)) !== JSON.stringify(identity),
679 );
680 const settingsChanged =
681 (news.nodeRecovery !== undefined && news.nodeRecovery !== described.NodeRecovery) ||
682 (news.nodeProvisioningMode !== undefined &&
683 news.nodeProvisioningMode !== described.NodeProvisioningMode) ||
684 (news.tieredStorageConfig !== undefined &&
685 JSON.stringify(news.tieredStorageConfig) !==
686 JSON.stringify(described.TieredStorageConfig)) ||
687 (news.autoScaling !== undefined &&
688 news.autoScaling.Mode !== described.AutoScaling?.Mode);
689
690 if (groupsChanged || settingsChanged) {
691 yield* sagemaker
692 .updateCluster({
693 ClusterName: name,
694 InstanceGroups: toGroupSpecs(news.instanceGroups),
695 RestrictedInstanceGroups: toGroupSpecs(news.restrictedInstanceGroups),
696 RestrictedInstanceGroupsConfig: news.restrictedInstanceGroupsConfig,
697 TieredStorageConfig: news.tieredStorageConfig,
698 NodeRecovery: news.nodeRecovery,
699 InstanceGroupsToDelete: groupsToDelete.length > 0 ? groupsToDelete : undefined,
700 NodeProvisioningMode: news.nodeProvisioningMode,
701 ClusterRole: news.clusterRole,
702 AutoScaling: news.autoScaling,
703 })
704 .pipe(Effect.catchTag("ResourceNotFound", () => Effect.void));
705 yield* session.note(`Updating HyperPod cluster ${name}...`);
706 }
707 }
708
709 // Converge to InService (provisioning is asynchronous).
710 yield* waitForCluster(name, "InService");
711 described = yield* describeClusterOrUndefined(name);
712 if (described === undefined) {
713 return yield* new SageMakerClusterUnexpectedResponse({
714 message: `failed to read reconciled HyperPod cluster ${name}`,
715 });
716 }
717 const attrs = toAttrs(described);
718
719 // Sync tags — diff against OBSERVED cloud tags.
720 const currentTags = yield* fetchClusterTags(attrs.clusterArn);
721 const { removed, upsert } = diffTags(currentTags, desiredTags);
722 if (removed.length > 0) {
723 yield* sagemaker.deleteTags({
724 ResourceArn: attrs.clusterArn,
725 TagKeys: removed,
726 });
727 }
728 if (upsert.length > 0) {
729 yield* sagemaker.addTags({
730 ResourceArn: attrs.clusterArn,
731 Tags: upsert.map(({ Key, Value }) => ({ Key, Value })),
732 });
733 }
734
735 yield* session.note(attrs.clusterArn);
736 return attrs;
737 }),
738 delete: Effect.fn(function* ({ output }) {
739 const described = yield* describeClusterOrUndefined(output.clusterName);
740 if (described === undefined) return;
741 // A cluster mid-create/update rejects deletion with a Conflict —
742 // wait for it to settle first, then delete. A cluster stuck in
743 // `Failed` (or one that never converges) is still deletable. One
744 // already `Deleting` (a previous attempt) skips straight to the
745 // Gone wait instead of burning the full InService wait budget.
746 if (described.ClusterStatus !== "Deleting") {
747 yield* waitForCluster(output.clusterName, "InService").pipe(
748 Effect.catchTag(["ClusterFailed", "ClusterNotReady"], () => Effect.void),
749 );
750 yield* sagemaker.deleteCluster({ ClusterName: output.clusterName }).pipe(
751 Effect.catchTags({
752 ResourceNotFound: () => Effect.void,
753 ConflictException: () => Effect.void,
754 }),
755 );
756 }
757 yield* waitForCluster(output.clusterName, "Gone");
758 }),
759 };
760 }),
761 );