The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as sagemaker from "@distilled.cloud/aws/sagemaker"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import type { Input } from "alchemy/Input"; |
| 5 | import * as Output from "alchemy/Output"; |
| 6 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 7 | import * as Provider from "alchemy/Provider"; |
| 8 | import { Resource } from "alchemy/Resource"; |
| 9 | import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags"; |
| 10 | import * as Data from "effect/Data"; |
| 11 | import * as Effect from "effect/Effect"; |
| 12 | import * as Schedule from "effect/Schedule"; |
| 13 | import * as EffectStream from "effect/Stream"; |
| 14 | import type { Providers } from "../Providers.ts"; |
| 15 | |
| 16 | export class SageMakerClusterInvalidProps extends Data.TaggedError("SageMakerClusterInvalidProps")<{ |
| 17 | readonly message: string; |
| 18 | readonly cause?: unknown; |
| 19 | }> {} |
| 20 | |
| 21 | export class SageMakerClusterUnexpectedResponse extends Data.TaggedError( |
| 22 | "SageMakerClusterUnexpectedResponse", |
| 23 | )<{ |
| 24 | readonly message: string; |
| 25 | readonly cause?: unknown; |
| 26 | }> {} |
| 27 | |
| 28 | export type ClusterStatus = sagemaker.ClusterStatus; |
| 29 | |
| 30 | /** |
| 31 | * The well-known node label carrying the HyperPod instance-group name. |
| 32 | * HyperPod nodes join the orchestrating EKS cluster as ordinary Kubernetes |
| 33 | * nodes carrying this label. |
| 34 | */ |
| 35 | export const HYPERPOD_INSTANCE_GROUP_LABEL = "sagemaker.amazonaws.com/instance-group-name"; |
| 36 | |
| 37 | /** The well-known node label carrying HyperPod's node health verdict. */ |
| 38 | export const HYPERPOD_NODE_HEALTH_LABEL = "sagemaker.amazonaws.com/node-health-status"; |
| 39 | |
| 40 | /** |
| 41 | * An instance group surfaced on the cluster's attributes — Kubernetes |
| 42 | * workloads pin themselves to the group by referencing its |
| 43 | * {@link ClusterInstanceGroupRef.nodeSelector}. |
| 44 | */ |
| 45 | export interface ClusterInstanceGroupRef { |
| 46 | /** The group's name (the `sagemaker.amazonaws.com/instance-group-name` node label). */ |
| 47 | InstanceGroupName: string; |
| 48 | /** The group's instance type (e.g. `ml.g5.xlarge`). */ |
| 49 | InstanceType: string | undefined; |
| 50 | /** Instances currently in service. */ |
| 51 | CurrentCount: number | undefined; |
| 52 | /** Instances the group is converging toward. */ |
| 53 | TargetCount: number | undefined; |
| 54 | /** |
| 55 | * A Kubernetes node selector pinning pods onto this group's |
| 56 | * health-checked nodes — pass it to a `Kubernetes.Job` / |
| 57 | * `Kubernetes.Deployment` through the `podTemplate` escape hatch: |
| 58 | * |
| 59 | * ```ts |
| 60 | * podTemplate: { |
| 61 | * spec: { nodeSelector: hyperpod.instanceGroups.workers.nodeSelector }, |
| 62 | * } |
| 63 | * ``` |
| 64 | */ |
| 65 | nodeSelector: Record<string, string>; |
| 66 | } |
| 67 | |
| 68 | /** |
| 69 | * One instance group's specification, sans name — the group's name is the |
| 70 | * key in {@link ClusterProps.instanceGroups}. |
| 71 | */ |
| 72 | export type ClusterInstanceGroup = Omit< |
| 73 | sagemaker.ClusterInstanceGroupSpecification, |
| 74 | "InstanceGroupName" |
| 75 | >; |
| 76 | |
| 77 | /** |
| 78 | * One restricted instance group's specification, sans name — the group's |
| 79 | * name is the key in {@link ClusterProps.restrictedInstanceGroups}. |
| 80 | */ |
| 81 | export type ClusterRestrictedInstanceGroup = Omit< |
| 82 | sagemaker.ClusterRestrictedInstanceGroupSpecification, |
| 83 | "InstanceGroupName" |
| 84 | >; |
| 85 | |
| 86 | export interface ClusterProps { |
| 87 | /** |
| 88 | * Name of the HyperPod cluster. Maximum 63 characters, alphanumeric and |
| 89 | * hyphens. |
| 90 | * @default ${app}-${stage}-${id} |
| 91 | */ |
| 92 | clusterName?: string; |
| 93 | /** |
| 94 | * The instance groups of the cluster, keyed by group name. Each group |
| 95 | * specifies its instance type, instance count, lifecycle config (S3 URI |
| 96 | * + `on_create` script) and execution role. |
| 97 | * |
| 98 | * Groups are updated in place; removing a key deletes the group from the |
| 99 | * cluster. The keys carry through to the cluster's `instanceGroups` |
| 100 | * attribute, so `hyperpod.instanceGroups.workers` is typed per key. |
| 101 | */ |
| 102 | instanceGroups?: Record<string, ClusterInstanceGroup>; |
| 103 | /** |
| 104 | * Restricted instance groups for HyperPod clusters running Amazon-managed |
| 105 | * workloads (e.g. Nova model customization), keyed by group name. |
| 106 | */ |
| 107 | restrictedInstanceGroups?: Record<string, ClusterRestrictedInstanceGroup>; |
| 108 | /** |
| 109 | * Shared environment configuration (e.g. FSx for Lustre) for restricted |
| 110 | * instance groups. |
| 111 | */ |
| 112 | restrictedInstanceGroupsConfig?: sagemaker.ClusterRestrictedInstanceGroupsConfig; |
| 113 | /** |
| 114 | * VPC to launch the cluster's nodes into. Changing the VPC replaces the |
| 115 | * cluster. |
| 116 | */ |
| 117 | vpcConfig?: sagemaker.VpcConfig; |
| 118 | /** |
| 119 | * The cluster's orchestrator. Set `{ Eks: { ClusterArn } }` to attach the |
| 120 | * HyperPod cluster to an EKS control plane; defaults to Slurm when |
| 121 | * omitted. |
| 122 | */ |
| 123 | orchestrator?: sagemaker.ClusterOrchestrator; |
| 124 | /** |
| 125 | * Whether HyperPod automatically replaces faulty nodes. |
| 126 | * @default "Automatic" |
| 127 | */ |
| 128 | nodeRecovery?: sagemaker.ClusterNodeRecovery; |
| 129 | /** |
| 130 | * Managed tiered storage (memory + local NVMe) configuration for |
| 131 | * checkpointing. |
| 132 | */ |
| 133 | tieredStorageConfig?: sagemaker.ClusterTieredStorageConfig; |
| 134 | /** |
| 135 | * Set to `"Continuous"` to enable continuous node provisioning, where |
| 136 | * HyperPod provisions capacity as it becomes available instead of |
| 137 | * all-or-nothing. |
| 138 | */ |
| 139 | nodeProvisioningMode?: sagemaker.ClusterNodeProvisioningMode; |
| 140 | /** |
| 141 | * ARN of the IAM role HyperPod assumes for cluster-level operations |
| 142 | * (required for continuous provisioning and autoscaling). |
| 143 | */ |
| 144 | clusterRole?: string; |
| 145 | /** |
| 146 | * Cluster autoscaling configuration (e.g. Karpenter) for EKS-orchestrated |
| 147 | * clusters. |
| 148 | */ |
| 149 | autoScaling?: sagemaker.ClusterAutoScalingConfig; |
| 150 | /** |
| 151 | * Tags to associate with the cluster. Merged with internal Alchemy tags. |
| 152 | */ |
| 153 | tags?: Record<string, string>; |
| 154 | } |
| 155 | |
| 156 | export interface Cluster extends Resource< |
| 157 | "AWS.SageMaker.Cluster", |
| 158 | ClusterProps, |
| 159 | { |
| 160 | /** |
| 161 | * The cluster's name. |
| 162 | */ |
| 163 | clusterName: string; |
| 164 | /** |
| 165 | * ARN of the cluster. |
| 166 | */ |
| 167 | clusterArn: string; |
| 168 | /** |
| 169 | * The cluster's status (`InService` once reconciled). |
| 170 | */ |
| 171 | clusterStatus: ClusterStatus; |
| 172 | /** |
| 173 | * ARN of the orchestrating EKS cluster, when EKS-orchestrated. |
| 174 | */ |
| 175 | orchestratorEksClusterArn: string | undefined; |
| 176 | /** |
| 177 | * The cluster's instance groups, keyed by group name. Pin a |
| 178 | * Kubernetes workload onto a group through the resource graph with |
| 179 | * its node selector: |
| 180 | * `hyperpod.instanceGroups.workers.nodeSelector`. |
| 181 | */ |
| 182 | instanceGroups: Record<string, ClusterInstanceGroupRef>; |
| 183 | }, |
| 184 | never, |
| 185 | Providers |
| 186 | > {} |
| 187 | |
| 188 | const ClusterResource = Resource<Cluster>("AWS.SageMaker.Cluster"); |
| 189 | |
| 190 | /** |
| 191 | * A `Cluster` narrowed to the instance-group keys declared in its props — |
| 192 | * `hyperpod.instanceGroups.workers` is typed per key, and a typo'd key is |
| 193 | * a compile error. |
| 194 | */ |
| 195 | export type ClusterOf<Groups> = Omit<Cluster, "instanceGroups"> & { |
| 196 | instanceGroups: Output.ObjectExpr<{ [K in keyof Groups]: ClusterInstanceGroupRef }, never>; |
| 197 | }; |
| 198 | |
| 199 | /** |
| 200 | * An Amazon SageMaker HyperPod cluster — a resilient, persistent cluster of |
| 201 | * ML compute for distributed training and inference, orchestrated by Slurm |
| 202 | * or EKS, with automatic faulty-node recovery and deep health checks. |
| 203 | * |
| 204 | * Provisioning a HyperPod cluster takes 10–25 minutes; instance groups are |
| 205 | * updated in place and removing a group from `instanceGroups` deletes it |
| 206 | * from the cluster. |
| 207 | * ### Creating Clusters |
| 208 | * **Example:** Slurm-Orchestrated Cluster |
| 209 | * ```typescript |
| 210 | * import * as AWS from "@alchemy.run/aws"; |
| 211 | * |
| 212 | * const cluster = yield* AWS.SageMaker.Cluster("TrainingCluster", { |
| 213 | * instanceGroups: { |
| 214 | * controller: { |
| 215 | * InstanceType: "ml.t3.medium", |
| 216 | * InstanceCount: 1, |
| 217 | * ExecutionRole: role.roleArn, |
| 218 | * // HyperPod now provisions Slurm clusters in Continuous mode, which |
| 219 | * // requires a Slurm node type on at least one instance group. |
| 220 | * SlurmConfig: { NodeType: "Controller" }, |
| 221 | * LifeCycleConfig: { |
| 222 | * SourceS3Uri: `s3://${bucket.bucketName}/lifecycle`, |
| 223 | * OnCreate: "on_create.sh", |
| 224 | * }, |
| 225 | * }, |
| 226 | * }, |
| 227 | * }); |
| 228 | * ``` |
| 229 | * |
| 230 | * **Example:** EKS-Orchestrated Cluster |
| 231 | * ```typescript |
| 232 | * // The EKS cluster must use the `API` (or `API_AND_CONFIG_MAP`) |
| 233 | * // authentication mode — pass `accessConfig` explicitly, EKS's own |
| 234 | * // CONFIG_MAP default is rejected. LifeCycleConfig is required for |
| 235 | * // EKS-orchestrated instance groups too. |
| 236 | * const hyperpod = yield* AWS.SageMaker.Cluster("EksHyperPod", { |
| 237 | * orchestrator: { Eks: { ClusterArn: eksCluster.clusterArn } }, |
| 238 | * vpcConfig: { |
| 239 | * SecurityGroupIds: [securityGroupId], |
| 240 | * Subnets: network.privateSubnetIds, |
| 241 | * }, |
| 242 | * instanceGroups: { |
| 243 | * workers: { |
| 244 | * InstanceType: "ml.g5.xlarge", |
| 245 | * InstanceCount: 2, |
| 246 | * ExecutionRole: role.roleArn, |
| 247 | * LifeCycleConfig: { |
| 248 | * SourceS3Uri: `s3://${bucket.bucketName}/lifecycle`, |
| 249 | * OnCreate: "on_create.sh", |
| 250 | * }, |
| 251 | * }, |
| 252 | * }, |
| 253 | * nodeRecovery: "Automatic", |
| 254 | * }); |
| 255 | * |
| 256 | * // The keys carry through to the attributes — typed per key: |
| 257 | * const workers = hyperpod.instanceGroups.workers; |
| 258 | * ``` |
| 259 | * |
| 260 | * ### Running Workloads (Slurm) |
| 261 | * **Example:** Submit jobs from the login node over SSM |
| 262 | * ```sh |
| 263 | * # Slurm jobs are submitted on the cluster itself. Each node is an SSM |
| 264 | * # target named sagemaker-cluster:<cluster-id>_<instance-group>-<instance-id> |
| 265 | * # (list nodes with `aws sagemaker list-cluster-nodes`). |
| 266 | * aws ssm start-session \ |
| 267 | * --target sagemaker-cluster:6wl4at0i68c6_controller-i-0123456789abcdef0 |
| 268 | * # then, on the node: |
| 269 | * sbatch --nodes=4 train.sbatch |
| 270 | * ``` |
| 271 | * |
| 272 | * ### Running Workloads (EKS) |
| 273 | * **Example:** Low level: apply any Kubernetes manifest to the orchestrator |
| 274 | * ```typescript |
| 275 | * // HyperPod nodes are ordinary EKS nodes — target them from a raw |
| 276 | * // manifest (a PyTorchJob CRD, a batch/v1 Job, ...) with the well-known |
| 277 | * // node labels. |
| 278 | * const job = yield* AWS.EKS.Manifest("RawTrainJob", { |
| 279 | * cluster: eksCluster, |
| 280 | * manifest: { |
| 281 | * apiVersion: "batch/v1", |
| 282 | * kind: "Job", |
| 283 | * metadata: { name: "raw-train", namespace: "default" }, |
| 284 | * spec: { |
| 285 | * template: { |
| 286 | * spec: { |
| 287 | * nodeSelector: { |
| 288 | * "sagemaker.amazonaws.com/node-health-status": "Schedulable", |
| 289 | * "sagemaker.amazonaws.com/instance-group-name": "workers", |
| 290 | * }, |
| 291 | * containers: [{ name: "train", image: "ghcr.io/acme/train:v3" }], |
| 292 | * restartPolicy: "Never", |
| 293 | * }, |
| 294 | * }, |
| 295 | * }, |
| 296 | * }, |
| 297 | * }); |
| 298 | * ``` |
| 299 | * |
| 300 | * **Example:** High level: an effectful Job pinned to HyperPod nodes |
| 301 | * ```typescript |
| 302 | * // Kubernetes.Job / Kubernetes.Deployment run on HyperPod via the |
| 303 | * // orchestrating EKS cluster in plain Kubernetes vocabulary — the |
| 304 | * // HyperPod resources expose the derived values as attributes: the |
| 305 | * // group's `nodeSelector`, the quota's governed `namespace` and Kueue |
| 306 | * // `queueName`. |
| 307 | * const evaluate = yield* Kubernetes.Job( |
| 308 | * "Evaluate", |
| 309 | * { |
| 310 | * cluster: eksCluster, |
| 311 | * main: import.meta.url, |
| 312 | * namespace: quota.namespace, |
| 313 | * labels: { |
| 314 | * [AWS.SageMaker.KUEUE_QUEUE_NAME_LABEL]: quota.queueName, |
| 315 | * [AWS.SageMaker.KUEUE_PRIORITY_CLASS_LABEL]: "training-priority", |
| 316 | * }, |
| 317 | * podTemplate: { |
| 318 | * spec: { |
| 319 | * nodeSelector: hyperpod.instanceGroups.workers.nodeSelector, |
| 320 | * }, |
| 321 | * }, |
| 322 | * }, |
| 323 | * Effect.gen(function* () { |
| 324 | * const putItem = yield* AWS.DynamoDB.PutItem(resultsTable); |
| 325 | * return { |
| 326 | * run: Effect.gen(function* () { |
| 327 | * // evaluation logic; bindings land IAM on the pod-identity role |
| 328 | * }), |
| 329 | * }; |
| 330 | * }).pipe(Effect.provide(AWS.DynamoDB.PutItemHttp)), |
| 331 | * ); |
| 332 | * ``` |
| 333 | * |
| 334 | * ### Task Governance |
| 335 | * **Example:** Prioritize workloads with a scheduler policy and team quotas |
| 336 | * ```typescript |
| 337 | * // Requires the amazon-sagemaker-hyperpod-taskgovernance EKS add-on. |
| 338 | * const policy = yield* AWS.SageMaker.ClusterSchedulerConfig("Scheduler", { |
| 339 | * clusterArn: hyperpod.clusterArn, |
| 340 | * schedulerConfig: { |
| 341 | * PriorityClasses: [{ Name: "training", Weight: 90 }], |
| 342 | * FairShare: "Enabled", |
| 343 | * }, |
| 344 | * }); |
| 345 | * |
| 346 | * // Creates the hyperpod-ns-research namespace + Kueue LocalQueue — |
| 347 | * // exposed as `quota.namespace` / `quota.queueName` for governed |
| 348 | * // Kubernetes workloads to reference. |
| 349 | * const quota = yield* AWS.SageMaker.ComputeQuota("ResearchQuota", { |
| 350 | * clusterArn: hyperpod.clusterArn, |
| 351 | * computeQuotaTarget: { TeamName: "research", FairShareWeight: 10 }, |
| 352 | * computeQuotaConfig: { |
| 353 | * ComputeQuotaResources: [ |
| 354 | * { InstanceType: "ml.g5.xlarge", Count: 1 }, |
| 355 | * ], |
| 356 | * }, |
| 357 | * }); |
| 358 | * ``` |
| 359 | * |
| 360 | * @resource |
| 361 | */ |
| 362 | export const Cluster: { |
| 363 | < |
| 364 | const Props extends { |
| 365 | [prop in keyof ClusterProps]: Input<ClusterProps[prop]>; |
| 366 | }, |
| 367 | >( |
| 368 | id: string, |
| 369 | props: Props | Effect.Effect<Props>, |
| 370 | ): Effect.Effect<ClusterOf<NonNullable<Props["instanceGroups"]>>, never, Providers>; |
| 371 | } & typeof ClusterResource = ClusterResource as never; |
| 372 | |
| 373 | const createClusterName = (id: string, props: { clusterName?: string | undefined }) => |
| 374 | props.clusterName ? Effect.succeed(props.clusterName) : createPhysicalName({ id, maxLength: 63 }); |
| 375 | |
| 376 | const describeClusterOrUndefined = (name: string) => |
| 377 | sagemaker |
| 378 | .describeCluster({ ClusterName: name }) |
| 379 | .pipe(Effect.catchTag("ResourceNotFound", () => Effect.undefined)); |
| 380 | |
| 381 | const fetchClusterTags = Effect.fn(function* (arn: string) { |
| 382 | const response = yield* sagemaker |
| 383 | .listTags({ ResourceArn: arn }) |
| 384 | .pipe(Effect.catchTag("AccessDeniedException", () => Effect.undefined)); |
| 385 | return Object.fromEntries( |
| 386 | (response?.Tags ?? []).flatMap((tag) => |
| 387 | tag.Key !== undefined ? [[tag.Key, tag.Value ?? ""]] : [], |
| 388 | ), |
| 389 | ); |
| 390 | }); |
| 391 | |
| 392 | const toAttrs = (described: sagemaker.DescribeClusterResponse): Cluster["Attributes"] => ({ |
| 393 | clusterName: described.ClusterName ?? "", |
| 394 | clusterArn: described.ClusterArn, |
| 395 | clusterStatus: described.ClusterStatus, |
| 396 | orchestratorEksClusterArn: described.Orchestrator?.Eks?.ClusterArn, |
| 397 | instanceGroups: Object.fromEntries( |
| 398 | (described.InstanceGroups ?? []).flatMap((group) => |
| 399 | group.InstanceGroupName !== undefined |
| 400 | ? [ |
| 401 | [ |
| 402 | group.InstanceGroupName, |
| 403 | { |
| 404 | InstanceGroupName: group.InstanceGroupName, |
| 405 | InstanceType: group.InstanceType, |
| 406 | CurrentCount: group.CurrentCount, |
| 407 | TargetCount: group.TargetCount, |
| 408 | nodeSelector: { |
| 409 | [HYPERPOD_NODE_HEALTH_LABEL]: "Schedulable", |
| 410 | [HYPERPOD_INSTANCE_GROUP_LABEL]: group.InstanceGroupName, |
| 411 | }, |
| 412 | }, |
| 413 | ], |
| 414 | ] |
| 415 | : [], |
| 416 | ), |
| 417 | ), |
| 418 | }); |
| 419 | |
| 420 | /** |
| 421 | * The cluster is still transitioning toward the awaited state — retried by |
| 422 | * the bounded wait schedule. |
| 423 | */ |
| 424 | class ClusterNotReady extends Data.TaggedError("ClusterNotReady")<{ |
| 425 | readonly clusterName: string; |
| 426 | readonly status: string | undefined; |
| 427 | }> {} |
| 428 | |
| 429 | /** |
| 430 | * The cluster converged to the terminal `Failed` status. |
| 431 | */ |
| 432 | export class ClusterFailed extends Data.TaggedError("ClusterFailed")<{ |
| 433 | readonly clusterName: string; |
| 434 | readonly message: string | undefined; |
| 435 | }> {} |
| 436 | |
| 437 | /** |
| 438 | * The cluster is stuck in `Deleting` because SageMaker's internal teardown |
| 439 | * cannot proceed — e.g. the instance group's execution role (which HyperPod |
| 440 | * assumes to delete node ENIs) was deleted or lost its |
| 441 | * `ec2:DeleteNetworkInterface` / `ec2:DeleteNetworkInterfacePermission` |
| 442 | * permissions mid-teardown. Retrying `deleteCluster` can never fix this; |
| 443 | * the role and its permissions must be restored out-of-band, after which |
| 444 | * SageMaker's own retry completes the deletion within a minute. |
| 445 | */ |
| 446 | export class ClusterTeardownBlocked extends Data.TaggedError("ClusterTeardownBlocked")<{ |
| 447 | readonly clusterName: string; |
| 448 | readonly message: string; |
| 449 | }> {} |
| 450 | |
| 451 | // Explicitly-typed retry wrapper — an inline `Effect.retry` in provider |
| 452 | // lifecycle code leaks `Retry.Return`'s conditional type into declaration |
| 453 | // emit and widens the provider layer to `unknown` for every consumer of |
| 454 | // `AWS.providers()`. |
| 455 | const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>( |
| 456 | self: Effect.Effect<A, E, R>, |
| 457 | ): Effect.Effect<A, E, R> => |
| 458 | Effect.retry(self, { |
| 459 | while: (e) => e._tag === "ClusterNotReady", |
| 460 | // HyperPod cluster provisioning routinely takes 10–25 minutes. |
| 461 | // Poll 15s up to ~35 min. |
| 462 | schedule: Schedule.max([Schedule.spaced("15 seconds"), Schedule.recurs(140)]), |
| 463 | }); |
| 464 | |
| 465 | // A freshly created execution role isn't assumable by SageMaker for a few |
| 466 | // seconds (IAM propagation) — createCluster rejects it with a |
| 467 | // ValidationException until it is. |
| 468 | const retryWhileRoleUnassumable = < |
| 469 | A, |
| 470 | E extends { readonly _tag: string; readonly message?: string }, |
| 471 | R, |
| 472 | >( |
| 473 | self: Effect.Effect<A, E, R>, |
| 474 | ): Effect.Effect<A, E, R> => |
| 475 | Effect.retry(self, { |
| 476 | while: (e): boolean => |
| 477 | e._tag === "ValidationException" && |
| 478 | (e.message?.includes("cannot assume the execution role") ?? false), |
| 479 | schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(12)]), |
| 480 | }); |
| 481 | |
| 482 | const waitForCluster = (name: string, target: "InService" | "Gone") => |
| 483 | retryWhileNotReady( |
| 484 | Effect.gen(function* () { |
| 485 | const described = yield* describeClusterOrUndefined(name); |
| 486 | if (target === "Gone") { |
| 487 | if (described === undefined) return; |
| 488 | // SageMaker's internal teardown deletes node ENIs by assuming the |
| 489 | // instance group's execution role. If that role (or its EC2 |
| 490 | // permissions) was deleted mid-teardown, every internal retry |
| 491 | // fails and the cluster stays `Deleting` forever — surface the |
| 492 | // node's permission failure immediately instead of polling the |
| 493 | // full wait budget on a delete that can never finish. |
| 494 | const nodes = yield* sagemaker |
| 495 | .listClusterNodes({ ClusterName: name }) |
| 496 | .pipe(Effect.catchTag("ResourceNotFound", () => Effect.undefined)); |
| 497 | const blocked = (nodes?.ClusterNodeSummaries ?? []).find((node) => |
| 498 | node.InstanceStatus?.Message?.includes("does not have permission to perform"), |
| 499 | ); |
| 500 | if (blocked !== undefined) { |
| 501 | return yield* new ClusterTeardownBlocked({ |
| 502 | clusterName: name, |
| 503 | message: blocked.InstanceStatus?.Message ?? "", |
| 504 | }); |
| 505 | } |
| 506 | return yield* new ClusterNotReady({ |
| 507 | clusterName: name, |
| 508 | status: described.ClusterStatus, |
| 509 | }); |
| 510 | } |
| 511 | if (described?.ClusterStatus === "InService") return; |
| 512 | if (described?.ClusterStatus === "Failed") { |
| 513 | return yield* new ClusterFailed({ |
| 514 | clusterName: name, |
| 515 | message: described.FailureMessage, |
| 516 | }); |
| 517 | } |
| 518 | return yield* new ClusterNotReady({ |
| 519 | clusterName: name, |
| 520 | status: described?.ClusterStatus, |
| 521 | }); |
| 522 | }), |
| 523 | ); |
| 524 | |
| 525 | /** |
| 526 | * The subset of an instance group's specification that `updateCluster` |
| 527 | * can change and `describeCluster` reports back — used to diff desired |
| 528 | * groups against observed groups. |
| 529 | */ |
| 530 | const observedGroupIdentity = ( |
| 531 | group: sagemaker.DescribeClusterResponse["InstanceGroups"][number], |
| 532 | ) => ({ |
| 533 | count: group.TargetCount ?? group.CurrentCount, |
| 534 | type: group.InstanceType, |
| 535 | }); |
| 536 | |
| 537 | const desiredGroupIdentity = (group: ClusterInstanceGroup) => ({ |
| 538 | count: group.InstanceCount, |
| 539 | type: group.InstanceType, |
| 540 | }); |
| 541 | |
| 542 | /** Fold the keyed props form into the API's named-array form. */ |
| 543 | const toGroupSpecs = <G extends { InstanceGroupName?: string }>( |
| 544 | groups: Record<string, Omit<G, "InstanceGroupName">> | undefined, |
| 545 | ): G[] | undefined => |
| 546 | groups === undefined |
| 547 | ? undefined |
| 548 | : Object.entries(groups).map(([name, group]) => ({ InstanceGroupName: name, ...group }) as G); |
| 549 | |
| 550 | export const ClusterProvider = () => |
| 551 | Provider.effect( |
| 552 | Cluster, |
| 553 | Effect.gen(function* () { |
| 554 | return { |
| 555 | stables: ["clusterName", "clusterArn"], |
| 556 | // HyperPod's internal teardown deletes node ENIs by assuming the |
| 557 | // instance group's execution role inside the cluster's VPC — |
| 558 | // deleting the role or network mid-teardown wedges the cluster in |
| 559 | // `Deleting` permanently, so nuke must fully delete clusters |
| 560 | // before touching these types. |
| 561 | nuke: { |
| 562 | dependsOn: [ |
| 563 | "AWS.IAM.Role", |
| 564 | "AWS.IAM.Policy", |
| 565 | "AWS.IAM.InstanceProfile", |
| 566 | "AWS.EC2.Vpc", |
| 567 | "AWS.EC2.Subnet", |
| 568 | "AWS.EC2.SecurityGroup", |
| 569 | ], |
| 570 | }, |
| 571 | list: () => |
| 572 | Effect.gen(function* () { |
| 573 | const summaries = yield* sagemaker.listClusters.pages({}).pipe( |
| 574 | EffectStream.runCollect, |
| 575 | Effect.map((chunk) => |
| 576 | Array.from(chunk).flatMap((page) => page.ClusterSummaries ?? []), |
| 577 | ), |
| 578 | ); |
| 579 | return summaries.flatMap((s) => |
| 580 | s.ClusterName !== undefined && s.ClusterArn !== undefined |
| 581 | ? [ |
| 582 | { |
| 583 | clusterName: s.ClusterName, |
| 584 | clusterArn: s.ClusterArn, |
| 585 | clusterStatus: s.ClusterStatus ?? "InService", |
| 586 | orchestratorEksClusterArn: undefined, |
| 587 | instanceGroups: {}, |
| 588 | }, |
| 589 | ] |
| 590 | : [], |
| 591 | ); |
| 592 | }), |
| 593 | read: Effect.fn(function* ({ id, olds, output }) { |
| 594 | const name = output?.clusterName ?? (yield* createClusterName(id, olds ?? {})); |
| 595 | const described = yield* describeClusterOrUndefined(name); |
| 596 | if (!described || described.ClusterStatus === "Deleting") { |
| 597 | return undefined; |
| 598 | } |
| 599 | const attrs = toAttrs(described); |
| 600 | const tags = yield* fetchClusterTags(attrs.clusterArn); |
| 601 | return (yield* hasAlchemyTags(id, tags as Tags)) ? attrs : Unowned(attrs); |
| 602 | }), |
| 603 | diff: Effect.fn(function* ({ id, news, olds }) { |
| 604 | if (!isResolved(news)) return; |
| 605 | if (olds === undefined) return; |
| 606 | const oldName = yield* createClusterName(id, olds); |
| 607 | const newName = yield* createClusterName(id, news); |
| 608 | // The VPC is fixed at creation — changing it (or the name) |
| 609 | // replaces the cluster. Everything else updates in place. |
| 610 | if ( |
| 611 | oldName !== newName || |
| 612 | JSON.stringify(olds.vpcConfig) !== JSON.stringify(news.vpcConfig) |
| 613 | ) { |
| 614 | return { action: "replace" } as const; |
| 615 | } |
| 616 | }), |
| 617 | reconcile: Effect.fn(function* ({ id, news, output, session }) { |
| 618 | if (!news) { |
| 619 | return yield* new SageMakerClusterInvalidProps({ |
| 620 | message: "SageMaker HyperPod Cluster requires props", |
| 621 | }); |
| 622 | } |
| 623 | const name = output?.clusterName ?? (yield* createClusterName(id, news)); |
| 624 | const internalTags = yield* createInternalTags(id); |
| 625 | const desiredTags = { ...internalTags, ...news.tags }; |
| 626 | |
| 627 | // Observe. |
| 628 | let described = yield* describeClusterOrUndefined(name); |
| 629 | |
| 630 | // Ensure — create if missing; tolerate the already-exists race. |
| 631 | if (described === undefined) { |
| 632 | yield* sagemaker |
| 633 | .createCluster({ |
| 634 | ClusterName: name, |
| 635 | InstanceGroups: toGroupSpecs(news.instanceGroups), |
| 636 | RestrictedInstanceGroups: toGroupSpecs(news.restrictedInstanceGroups), |
| 637 | RestrictedInstanceGroupsConfig: news.restrictedInstanceGroupsConfig, |
| 638 | VpcConfig: news.vpcConfig, |
| 639 | Orchestrator: news.orchestrator, |
| 640 | NodeRecovery: news.nodeRecovery, |
| 641 | TieredStorageConfig: news.tieredStorageConfig, |
| 642 | NodeProvisioningMode: news.nodeProvisioningMode, |
| 643 | ClusterRole: news.clusterRole, |
| 644 | AutoScaling: news.autoScaling, |
| 645 | Tags: Object.entries(desiredTags).map(([Key, Value]) => ({ |
| 646 | Key, |
| 647 | Value, |
| 648 | })), |
| 649 | }) |
| 650 | .pipe( |
| 651 | retryWhileRoleUnassumable, |
| 652 | Effect.catchTag("ResourceInUse", () => Effect.void), |
| 653 | ); |
| 654 | yield* session.note(`Creating HyperPod cluster ${name} (typically 10-25 minutes)...`); |
| 655 | } else { |
| 656 | // Sync — diff observed instance groups + mutable settings |
| 657 | // against desired and apply one update call for the delta. |
| 658 | const observedGroups = new Map( |
| 659 | (described.InstanceGroups ?? []).flatMap((g) => |
| 660 | g.InstanceGroupName !== undefined |
| 661 | ? [[g.InstanceGroupName, observedGroupIdentity(g)]] |
| 662 | : [], |
| 663 | ), |
| 664 | ); |
| 665 | const desiredGroups = new Map( |
| 666 | Object.entries(news.instanceGroups ?? {}).map(([groupName, g]) => [ |
| 667 | groupName, |
| 668 | desiredGroupIdentity(g), |
| 669 | ]), |
| 670 | ); |
| 671 | const groupsToDelete = [...observedGroups.keys()].filter( |
| 672 | (groupName) => !desiredGroups.has(groupName), |
| 673 | ); |
| 674 | const groupsChanged = |
| 675 | groupsToDelete.length > 0 || |
| 676 | [...desiredGroups].some( |
| 677 | ([groupName, identity]) => |
| 678 | JSON.stringify(observedGroups.get(groupName)) !== JSON.stringify(identity), |
| 679 | ); |
| 680 | const settingsChanged = |
| 681 | (news.nodeRecovery !== undefined && news.nodeRecovery !== described.NodeRecovery) || |
| 682 | (news.nodeProvisioningMode !== undefined && |
| 683 | news.nodeProvisioningMode !== described.NodeProvisioningMode) || |
| 684 | (news.tieredStorageConfig !== undefined && |
| 685 | JSON.stringify(news.tieredStorageConfig) !== |
| 686 | JSON.stringify(described.TieredStorageConfig)) || |
| 687 | (news.autoScaling !== undefined && |
| 688 | news.autoScaling.Mode !== described.AutoScaling?.Mode); |
| 689 | |
| 690 | if (groupsChanged || settingsChanged) { |
| 691 | yield* sagemaker |
| 692 | .updateCluster({ |
| 693 | ClusterName: name, |
| 694 | InstanceGroups: toGroupSpecs(news.instanceGroups), |
| 695 | RestrictedInstanceGroups: toGroupSpecs(news.restrictedInstanceGroups), |
| 696 | RestrictedInstanceGroupsConfig: news.restrictedInstanceGroupsConfig, |
| 697 | TieredStorageConfig: news.tieredStorageConfig, |
| 698 | NodeRecovery: news.nodeRecovery, |
| 699 | InstanceGroupsToDelete: groupsToDelete.length > 0 ? groupsToDelete : undefined, |
| 700 | NodeProvisioningMode: news.nodeProvisioningMode, |
| 701 | ClusterRole: news.clusterRole, |
| 702 | AutoScaling: news.autoScaling, |
| 703 | }) |
| 704 | .pipe(Effect.catchTag("ResourceNotFound", () => Effect.void)); |
| 705 | yield* session.note(`Updating HyperPod cluster ${name}...`); |
| 706 | } |
| 707 | } |
| 708 | |
| 709 | // Converge to InService (provisioning is asynchronous). |
| 710 | yield* waitForCluster(name, "InService"); |
| 711 | described = yield* describeClusterOrUndefined(name); |
| 712 | if (described === undefined) { |
| 713 | return yield* new SageMakerClusterUnexpectedResponse({ |
| 714 | message: `failed to read reconciled HyperPod cluster ${name}`, |
| 715 | }); |
| 716 | } |
| 717 | const attrs = toAttrs(described); |
| 718 | |
| 719 | // Sync tags — diff against OBSERVED cloud tags. |
| 720 | const currentTags = yield* fetchClusterTags(attrs.clusterArn); |
| 721 | const { removed, upsert } = diffTags(currentTags, desiredTags); |
| 722 | if (removed.length > 0) { |
| 723 | yield* sagemaker.deleteTags({ |
| 724 | ResourceArn: attrs.clusterArn, |
| 725 | TagKeys: removed, |
| 726 | }); |
| 727 | } |
| 728 | if (upsert.length > 0) { |
| 729 | yield* sagemaker.addTags({ |
| 730 | ResourceArn: attrs.clusterArn, |
| 731 | Tags: upsert.map(({ Key, Value }) => ({ Key, Value })), |
| 732 | }); |
| 733 | } |
| 734 | |
| 735 | yield* session.note(attrs.clusterArn); |
| 736 | return attrs; |
| 737 | }), |
| 738 | delete: Effect.fn(function* ({ output }) { |
| 739 | const described = yield* describeClusterOrUndefined(output.clusterName); |
| 740 | if (described === undefined) return; |
| 741 | // A cluster mid-create/update rejects deletion with a Conflict — |
| 742 | // wait for it to settle first, then delete. A cluster stuck in |
| 743 | // `Failed` (or one that never converges) is still deletable. One |
| 744 | // already `Deleting` (a previous attempt) skips straight to the |
| 745 | // Gone wait instead of burning the full InService wait budget. |
| 746 | if (described.ClusterStatus !== "Deleting") { |
| 747 | yield* waitForCluster(output.clusterName, "InService").pipe( |
| 748 | Effect.catchTag(["ClusterFailed", "ClusterNotReady"], () => Effect.void), |
| 749 | ); |
| 750 | yield* sagemaker.deleteCluster({ ClusterName: output.clusterName }).pipe( |
| 751 | Effect.catchTags({ |
| 752 | ResourceNotFound: () => Effect.void, |
| 753 | ConflictException: () => Effect.void, |
| 754 | }), |
| 755 | ); |
| 756 | } |
| 757 | yield* waitForCluster(output.clusterName, "Gone"); |
| 758 | }), |
| 759 | }; |
| 760 | }), |
| 761 | ); |