The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as emrc from "@distilled.cloud/aws/emr-containers"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Option from "effect/Option"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | import type { Providers } from "../Providers.ts"; |
| 13 | |
| 14 | export interface VirtualClusterContainerProvider { |
| 15 | /** |
| 16 | * The name of the underlying EKS cluster. Changing it replaces the virtual |
| 17 | * cluster. |
| 18 | */ |
| 19 | id: string; |
| 20 | /** |
| 21 | * The container provider type. |
| 22 | * @default "EKS" |
| 23 | */ |
| 24 | type?: "EKS"; |
| 25 | /** |
| 26 | * The namespace registration for the underlying EKS cluster. |
| 27 | */ |
| 28 | info?: { |
| 29 | eksInfo: { |
| 30 | /** |
| 31 | * The Kubernetes namespace the virtual cluster maps to. Changing it |
| 32 | * replaces the virtual cluster. |
| 33 | */ |
| 34 | namespace?: string; |
| 35 | /** |
| 36 | * A node label selector restricting which nodes run the jobs. |
| 37 | */ |
| 38 | nodeLabel?: string; |
| 39 | }; |
| 40 | }; |
| 41 | } |
| 42 | |
| 43 | export interface VirtualClusterProps { |
| 44 | /** |
| 45 | * Name of the virtual cluster (1-64 characters: letters, digits, `.` `_` |
| 46 | * `/` `#` `-`). Changing the name replaces the virtual cluster. |
| 47 | * @default a generated physical name |
| 48 | */ |
| 49 | virtualClusterName?: string; |
| 50 | /** |
| 51 | * The EKS cluster and namespace this virtual cluster maps to. Virtual |
| 52 | * clusters are immutable — any change replaces the virtual cluster. |
| 53 | */ |
| 54 | containerProvider: VirtualClusterContainerProvider; |
| 55 | /** |
| 56 | * The ID of an EMR containers security configuration to attach. Changing |
| 57 | * it replaces the virtual cluster. |
| 58 | */ |
| 59 | securityConfigurationId?: string; |
| 60 | /** |
| 61 | * Tags to apply to the virtual cluster. Merged with the internal Alchemy |
| 62 | * tags. |
| 63 | */ |
| 64 | tags?: Record<string, string>; |
| 65 | } |
| 66 | |
| 67 | export interface VirtualCluster extends Resource< |
| 68 | "AWS.EMRContainers.VirtualCluster", |
| 69 | VirtualClusterProps, |
| 70 | { |
| 71 | /** The ID of the virtual cluster. */ |
| 72 | virtualClusterId: string; |
| 73 | /** The name of the virtual cluster. */ |
| 74 | virtualClusterName: string; |
| 75 | /** The ARN of the virtual cluster. */ |
| 76 | virtualClusterArn: string; |
| 77 | /** The name of the EKS cluster backing the virtual cluster. */ |
| 78 | eksClusterName: string; |
| 79 | /** The virtual cluster state (e.g. `RUNNING`, `TERMINATING`). */ |
| 80 | state?: string; |
| 81 | }, |
| 82 | {}, |
| 83 | Providers |
| 84 | > {} |
| 85 | |
| 86 | /** |
| 87 | * An Amazon EMR on EKS virtual cluster — a registration of a Kubernetes |
| 88 | * namespace on an EKS cluster as an EMR job-submission target. Virtual |
| 89 | * clusters consume no resources themselves (no cost while idle); jobs |
| 90 | * submitted to the virtual cluster run as pods in the mapped namespace. |
| 91 | * |
| 92 | * The underlying EKS cluster must grant Amazon EMR on EKS access to the |
| 93 | * namespace (via EKS access entries when the cluster's authentication mode |
| 94 | * includes `API`, or the legacy `aws-auth` ConfigMap). Everything except tags |
| 95 | * is immutable — changes replace the virtual cluster. |
| 96 | * |
| 97 | * ### Creating Virtual Clusters |
| 98 | * **Example:** Register an EKS Namespace with EMR |
| 99 | * ```typescript |
| 100 | * import * as AWS from "@alchemy.run/aws"; |
| 101 | * |
| 102 | * const cluster = yield* AWS.EKS.Cluster("Cluster", { |
| 103 | * roleArn: clusterRole.roleArn, |
| 104 | * resourcesVpcConfig: { subnetIds }, |
| 105 | * accessConfig: { authenticationMode: "API_AND_CONFIG_MAP" }, |
| 106 | * }); |
| 107 | * |
| 108 | * const virtualCluster = yield* AWS.EMRContainers.VirtualCluster("Spark", { |
| 109 | * containerProvider: { |
| 110 | * id: cluster.clusterName, |
| 111 | * info: { eksInfo: { namespace: "emr" } }, |
| 112 | * }, |
| 113 | * }); |
| 114 | * // virtualCluster.virtualClusterId is passed to StartJobRun |
| 115 | * ``` |
| 116 | * |
| 117 | * @resource |
| 118 | */ |
| 119 | export const VirtualCluster = Resource<VirtualCluster>("AWS.EMRContainers.VirtualCluster"); |
| 120 | |
| 121 | export const VirtualClusterProvider = () => |
| 122 | Provider.effect( |
| 123 | VirtualCluster, |
| 124 | Effect.gen(function* () { |
| 125 | const createName = Effect.fn(function* ( |
| 126 | id: string, |
| 127 | props: { virtualClusterName?: string | undefined }, |
| 128 | ) { |
| 129 | return props.virtualClusterName ?? (yield* createPhysicalName({ id, maxLength: 64 })); |
| 130 | }); |
| 131 | |
| 132 | const toContainerProvider = ( |
| 133 | props: VirtualClusterContainerProvider, |
| 134 | ): emrc.ContainerProvider => ({ |
| 135 | type: props.type ?? "EKS", |
| 136 | id: props.id, |
| 137 | info: props.info, |
| 138 | }); |
| 139 | |
| 140 | const toAttributes = (vc: emrc.VirtualCluster) => ({ |
| 141 | virtualClusterId: vc.id!, |
| 142 | virtualClusterName: vc.name!, |
| 143 | virtualClusterArn: vc.arn!, |
| 144 | eksClusterName: vc.containerProvider?.id ?? "", |
| 145 | state: vc.state, |
| 146 | }); |
| 147 | |
| 148 | const observeById = Effect.fn(function* (id: string) { |
| 149 | return yield* emrc.describeVirtualCluster({ id }).pipe( |
| 150 | Effect.map((response) => response.virtualCluster), |
| 151 | Effect.catchTag( |
| 152 | ["ResourceNotFoundException", "ValidationException"], |
| 153 | // a malformed/unknown id is "not present", same as not-found |
| 154 | () => Effect.undefined, |
| 155 | ), |
| 156 | ); |
| 157 | }); |
| 158 | |
| 159 | // Terminated virtual clusters linger in list results; only a RUNNING |
| 160 | // (or transiently ARRESTED) one with our name is "present". |
| 161 | const live = (state: string | undefined) => state === "RUNNING" || state === "ARRESTED"; |
| 162 | |
| 163 | const observeByName = Effect.fn(function* (name: string) { |
| 164 | return yield* emrc.listVirtualClusters.items({ states: ["RUNNING", "ARRESTED"] }).pipe( |
| 165 | Stream.filter((vc) => vc.name === name), |
| 166 | Stream.runHead, |
| 167 | Effect.map(Option.getOrUndefined), |
| 168 | ); |
| 169 | }); |
| 170 | |
| 171 | const observe = Effect.fn(function* (id: string | undefined, name: string) { |
| 172 | const byId = id !== undefined ? yield* observeById(id) : undefined; |
| 173 | if (byId !== undefined && live(byId.state)) { |
| 174 | return byId; |
| 175 | } |
| 176 | return yield* observeByName(name); |
| 177 | }); |
| 178 | |
| 179 | const syncTags = Effect.fn(function* ( |
| 180 | vc: emrc.VirtualCluster, |
| 181 | desired: Record<string, string>, |
| 182 | ) { |
| 183 | // Diff against OBSERVED cloud tags (adoption may bring foreign tags). |
| 184 | const observed = Object.fromEntries( |
| 185 | Object.entries(vc.tags ?? {}).filter( |
| 186 | (entry): entry is [string, string] => entry[1] !== undefined, |
| 187 | ), |
| 188 | ); |
| 189 | const { upsert, removed } = diffTags(observed, desired); |
| 190 | if (upsert.length > 0) { |
| 191 | yield* emrc.tagResource({ |
| 192 | resourceArn: vc.arn!, |
| 193 | tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])), |
| 194 | }); |
| 195 | } |
| 196 | if (removed.length > 0) { |
| 197 | yield* emrc.untagResource({ |
| 198 | resourceArn: vc.arn!, |
| 199 | tagKeys: removed, |
| 200 | }); |
| 201 | } |
| 202 | }); |
| 203 | |
| 204 | return VirtualCluster.Provider.of({ |
| 205 | stables: ["virtualClusterId", "virtualClusterName", "virtualClusterArn", "eksClusterName"], |
| 206 | |
| 207 | list: () => |
| 208 | Effect.gen(function* () { |
| 209 | const pages = yield* emrc.listVirtualClusters |
| 210 | .pages({ states: ["RUNNING", "ARRESTED"] }) |
| 211 | .pipe(Stream.runCollect); |
| 212 | return Array.from(pages) |
| 213 | .flatMap((page) => page.virtualClusters ?? []) |
| 214 | .filter((vc) => vc.id !== undefined && vc.name !== undefined && vc.arn !== undefined) |
| 215 | .map(toAttributes); |
| 216 | }), |
| 217 | |
| 218 | read: Effect.fn(function* ({ id, olds, output }) { |
| 219 | const name = output?.virtualClusterName ?? (yield* createName(id, olds ?? {})); |
| 220 | const vc = yield* observe(output?.virtualClusterId, name); |
| 221 | if (vc?.id === undefined || vc.arn === undefined) { |
| 222 | return undefined; |
| 223 | } |
| 224 | const attrs = toAttributes(vc); |
| 225 | return (yield* hasAlchemyTags(id, vc.tags)) ? attrs : Unowned(attrs); |
| 226 | }), |
| 227 | |
| 228 | diff: Effect.fn(function* ({ id, news, olds }) { |
| 229 | if (!isResolved(news)) return undefined; |
| 230 | const oldName = yield* createName(id, olds); |
| 231 | const newName = yield* createName(id, news); |
| 232 | if ( |
| 233 | oldName !== newName || |
| 234 | JSON.stringify(toContainerProvider(olds.containerProvider)) !== |
| 235 | JSON.stringify(toContainerProvider(news.containerProvider)) || |
| 236 | olds.securityConfigurationId !== news.securityConfigurationId |
| 237 | ) { |
| 238 | // everything except tags is immutable |
| 239 | return { action: "replace" } as const; |
| 240 | } |
| 241 | }), |
| 242 | |
| 243 | reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) { |
| 244 | const name = output?.virtualClusterName ?? (yield* createName(id, news)); |
| 245 | const internalTags = yield* createInternalTags(id); |
| 246 | const desiredTags = { ...news.tags, ...internalTags }; |
| 247 | |
| 248 | // 1. OBSERVE — cloud state is authoritative; output is an id cache |
| 249 | let vc = yield* observe(output?.virtualClusterId, name); |
| 250 | |
| 251 | // 2. ENSURE — create if missing (synchronous; RUNNING immediately) |
| 252 | if (vc === undefined) { |
| 253 | const created = yield* emrc |
| 254 | .createVirtualCluster({ |
| 255 | // deterministic per instance: a retried create after a crashed |
| 256 | // reconcile never double-provisions |
| 257 | clientToken: instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy", |
| 258 | name, |
| 259 | containerProvider: toContainerProvider(news.containerProvider), |
| 260 | securityConfigurationId: news.securityConfigurationId, |
| 261 | tags: desiredTags, |
| 262 | }) |
| 263 | .pipe( |
| 264 | // A just-created EKS cluster rejects attachment while its |
| 265 | // connection (access configuration) is still being updated. |
| 266 | Effect.retry({ |
| 267 | while: (e) => e._tag === "EksClusterConnectionUpdating", |
| 268 | schedule: Schedule.spaced("10 seconds"), |
| 269 | times: 18, |
| 270 | }), |
| 271 | ); |
| 272 | vc = |
| 273 | created.id !== undefined |
| 274 | ? yield* emrc |
| 275 | .describeVirtualCluster({ id: created.id }) |
| 276 | .pipe(Effect.map((r) => r.virtualCluster)) |
| 277 | : yield* observeByName(name); |
| 278 | if (vc?.id === undefined) { |
| 279 | return yield* emrc.ResourceNotFoundException.make({ |
| 280 | message: `virtual cluster ${name} not visible after create`, |
| 281 | }); |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | // 3. SYNC TAGS — the only mutable aspect; diff observed vs desired |
| 286 | yield* syncTags(vc, desiredTags); |
| 287 | |
| 288 | yield* session.note(vc.id!); |
| 289 | return toAttributes(vc); |
| 290 | }), |
| 291 | |
| 292 | delete: Effect.fn(function* ({ output }) { |
| 293 | // deleteVirtualCluster is idempotent-by-observation: a terminated |
| 294 | // or missing virtual cluster reads as "not live" and is skipped |
| 295 | // (repeat deletes of a terminated id fail with ValidationException). |
| 296 | const vc = yield* emrc.describeVirtualCluster({ id: output.virtualClusterId }).pipe( |
| 297 | Effect.map((response) => response.virtualCluster), |
| 298 | Effect.catchTag( |
| 299 | ["ResourceNotFoundException", "ValidationException"], |
| 300 | () => Effect.undefined, |
| 301 | ), |
| 302 | ); |
| 303 | if (vc === undefined || !live(vc.state)) { |
| 304 | return; |
| 305 | } |
| 306 | yield* emrc |
| 307 | .deleteVirtualCluster({ id: output.virtualClusterId }) |
| 308 | .pipe(Effect.catchTag("ValidationException", () => Effect.void)); |
| 309 | }), |
| 310 | }); |
| 311 | }), |
| 312 | ); |